diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2f73c46481b..f92c551677d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -370,7 +370,6 @@ jobs: !:trino-phoenix5, !:trino-pinot, !:trino-postgresql, - !:trino-raptor-legacy, !:trino-redis, !:trino-redshift, !:trino-resource-group-managers, @@ -476,7 +475,6 @@ jobs: - { modules: plugin/trino-phoenix5 } - { modules: plugin/trino-pinot } - { modules: plugin/trino-postgresql } - - { modules: plugin/trino-raptor-legacy } - { modules: plugin/trino-redis } - { modules: plugin/trino-redshift } - { modules: plugin/trino-redshift, profile: cloud-tests } diff --git a/core/trino-server/src/main/provisio/trino.xml b/core/trino-server/src/main/provisio/trino.xml index e27ac02f7f38..5fd5c6e2415d 100644 --- a/core/trino-server/src/main/provisio/trino.xml +++ b/core/trino-server/src/main/provisio/trino.xml @@ -279,12 +279,6 @@ - - - - - - diff --git a/docs/release-template.md b/docs/release-template.md index d41cf1df4f37..50df150ce128 100644 --- a/docs/release-template.md +++ b/docs/release-template.md @@ -70,8 +70,6 @@ ## Prometheus connector -## Raptor connector - ## Redis connector ## Redshift connector diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OutputStreamOrcDataSink.java b/lib/trino-orc/src/main/java/io/trino/orc/OutputStreamOrcDataSink.java index c357bb330660..65925c8b56d4 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OutputStreamOrcDataSink.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OutputStreamOrcDataSink.java @@ -41,15 +41,6 @@ public static OutputStreamOrcDataSink create(TrinoOutputFile outputFile) return new OutputStreamOrcDataSink(outputFile.create(memoryContext), memoryContext); } - // Do not use this method, it is here only for io.trino.plugin.raptor.legacy.storage.OrcFileWriter.createOrcDataSink - // and it should be removed in the future - @Deprecated - public static OutputStreamOrcDataSink create(OutputStream outputStream) - throws IOException - { - return new OutputStreamOrcDataSink(outputStream, newSimpleAggregatedMemoryContext()); - } - private OutputStreamOrcDataSink(OutputStream outputStream, AggregatedMemoryContext memoryContext) { this.output = new OutputStreamSliceOutput(requireNonNull(outputStream, "outputStream is null")); diff --git a/plugin/trino-raptor-legacy/pom.xml b/plugin/trino-raptor-legacy/pom.xml deleted file mode 100644 index dde2b04ac386..000000000000 --- a/plugin/trino-raptor-legacy/pom.xml +++ /dev/null @@ -1,306 +0,0 @@ - - - 4.0.0 - - - io.trino - trino-root - 460-SNAPSHOT - ../../pom.xml - - - trino-raptor-legacy - trino-plugin - Trino - Raptor legacy connector - - - - com.google.errorprone - error_prone_annotations - true - - - - com.google.guava - guava - - - - com.google.inject - guice - - - - com.h2database - h2 - - - - com.mysql - mysql-connector-j - - - - io.airlift - bootstrap - - - - io.airlift - concurrent - - - - io.airlift - configuration - - - - io.airlift - http-client - - - - io.airlift - json - - - - io.airlift - log - - - - io.airlift - stats - - - - io.airlift - units - - - - io.trino - trino-cache - - - - io.trino - trino-memory-context - - - - io.trino - trino-orc - - - - io.trino - trino-plugin-toolkit - - - - it.unimi.dsi - fastutil - - - - jakarta.annotation - jakarta.annotation-api - - - - jakarta.validation - jakarta.validation-api - - - - joda-time - joda-time - - - - org.gaul - modernizer-maven-annotations - - - - org.jdbi - jdbi3-core - - - - org.jdbi - jdbi3-sqlobject - - - - org.weakref - jmxutils - - - - com.fasterxml.jackson.core - jackson-annotations - provided - - - - io.airlift - slice - provided - - - - io.opentelemetry - opentelemetry-api - provided - - - - io.opentelemetry - opentelemetry-context - provided - - - - io.trino - trino-spi - provided - - - - org.openjdk.jol - jol-core - provided - - - - io.airlift - log-manager - runtime - - - - io.airlift - node - runtime - - - - io.airlift - http-server - test - - - - io.airlift - jaxrs - test - - - - io.airlift - junit-extensions - test - - - - io.airlift - testing - test - - - - io.trino - trino-client - test - - - - io.trino - trino-main - test-jar - test - - - - io.trino - trino-main - test - - - - io.trino - trino-testing - test - - - - io.trino - trino-testing-services - test - - - - io.trino - trino-tpch - test - - - - io.trino.tpch - tpch - test - - - - jakarta.servlet - jakarta.servlet-api - test - - - - jakarta.ws.rs - jakarta.ws.rs-api - test - - - - org.assertj - assertj-core - test - - - - org.jetbrains - annotations - test - - - - org.junit.jupiter - junit-jupiter-api - test - - - - org.junit.jupiter - junit-jupiter-engine - test - - - - org.testcontainers - mysql - test - - - - org.testcontainers - testcontainers - test - - - diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/NodeSupplier.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/NodeSupplier.java deleted file mode 100644 index db05b7a04b16..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/NodeSupplier.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.spi.Node; - -import java.util.Set; - -public interface NodeSupplier -{ - Set getWorkerNodes(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketFunction.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketFunction.java deleted file mode 100644 index 5ed85dc5b892..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketFunction.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.airlift.slice.XxHash64; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.VarcharType.VARCHAR; - -public class RaptorBucketFunction - implements BucketFunction -{ - private final HashFunction[] functions; - private final int bucketCount; - - public RaptorBucketFunction(int bucketCount, List types) - { - checkArgument(bucketCount > 0, "bucketCount must be at least one"); - this.bucketCount = bucketCount; - this.functions = types.stream() - .map(RaptorBucketFunction::getHashFunction) - .toArray(HashFunction[]::new); - } - - @SuppressWarnings("NumericCastThatLosesPrecision") - @Override - public int getBucket(Page page, int position) - { - long hash = 0; - for (int i = 0; i < page.getChannelCount(); i++) { - Block block = page.getBlock(i); - long value = functions[i].hash(block, position); - hash = (hash * 31) + value; - } - int value = (int) (hash & Integer.MAX_VALUE); - return value % bucketCount; - } - - public static void validateBucketType(Type type) - { - getHashFunction(type); - } - - private static HashFunction getHashFunction(Type type) - { - if (type.equals(BIGINT)) { - return bigintHashFunction(); - } - if (type.equals(INTEGER)) { - return intHashFunction(); - } - if (type instanceof VarcharType) { - return varcharHashFunction(); - } - throw new TrinoException(NOT_SUPPORTED, "Bucketing is supported for bigint, integer and varchar, not " + type.getDisplayName()); - } - - private static HashFunction bigintHashFunction() - { - return (block, position) -> XxHash64.hash(BIGINT.getLong(block, position)); - } - - private static HashFunction intHashFunction() - { - return (block, position) -> XxHash64.hash(INTEGER.getInt(block, position)); - } - - private static HashFunction varcharHashFunction() - { - return (block, position) -> XxHash64.hash(VARCHAR.getSlice(block, position)); - } - - private interface HashFunction - { - long hash(Block block, int position); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateFunction.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateFunction.java deleted file mode 100644 index 0e50dcdb4c07..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateFunction.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.RowBlock; -import io.trino.spi.block.SqlRow; -import io.trino.spi.connector.BucketFunction; - -import static io.trino.spi.type.IntegerType.INTEGER; - -public class RaptorBucketedUpdateFunction - implements BucketFunction -{ - @Override - public int getBucket(Page page, int position) - { - Block block = page.getBlock(0); - SqlRow row = ((RowBlock) block.getUnderlyingValueBlock()).getRow(block.getUnderlyingValuePosition(position)); - return INTEGER.getInt(row.getRawFieldBlock(0), row.getRawIndex()); // bucket field of row ID - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateHandle.java deleted file mode 100644 index 2c0d7ff6a465..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateHandle.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; - -import java.util.List; - -public class RaptorBucketedUpdateHandle - extends RaptorPartitioningHandle -{ - @JsonCreator - public RaptorBucketedUpdateHandle(long distributionId, List bucketToNode) - { - super(distributionId, bucketToNode); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorColumnHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorColumnHandle.java deleted file mode 100644 index 7cf69e07665c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorColumnHandle.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.type.Type; - -import java.util.Objects; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RowType.field; -import static io.trino.spi.type.RowType.rowType; -import static io.trino.spi.type.UuidType.UUID; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static java.util.Objects.requireNonNull; - -public final class RaptorColumnHandle - implements ColumnHandle -{ - // Generated rowId column for updates - private static final long SHARD_ROW_ID_COLUMN_ID = -1; - - public static final long SHARD_UUID_COLUMN_ID = -2; - public static final String SHARD_UUID_COLUMN_NAME = "$shard_uuid"; - public static final Type SHARD_UUID_COLUMN_TYPE = createVarcharType(36); - - public static final long BUCKET_NUMBER_COLUMN_ID = -3; - public static final String BUCKET_NUMBER_COLUMN_NAME = "$bucket_number"; - - private static final long MERGE_ROW_ID_COLUMN_ID = -4; - private static final String MERGE_ROW_ID_COLUMN_NAME = "$merge_row_id"; - private static final Type MERGE_ROW_ID_COLUMN_TYPE = rowType( - field("bucket", INTEGER), - field("uuid", UUID), - field("row_id", BIGINT)); - - private final String columnName; - private final long columnId; - private final Type columnType; - - @JsonCreator - public RaptorColumnHandle( - @JsonProperty("columnName") String columnName, - @JsonProperty("columnId") long columnId, - @JsonProperty("columnType") Type columnType) - { - this.columnName = requireNonNull(columnName, "columnName is null"); - this.columnId = columnId; - this.columnType = requireNonNull(columnType, "columnType is null"); - } - - @JsonProperty - public String getColumnName() - { - return columnName; - } - - @JsonProperty - public long getColumnId() - { - return columnId; - } - - @JsonProperty - public Type getColumnType() - { - return columnType; - } - - @Override - public String toString() - { - return columnName + ":" + columnId + ":" + columnType; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - RaptorColumnHandle other = (RaptorColumnHandle) obj; - return this.columnId == other.columnId; - } - - @Override - public int hashCode() - { - return Objects.hash(columnId); - } - - public boolean isShardUuid() - { - return isShardUuidColumn(columnId); - } - - public boolean isBucketNumber() - { - return isBucketNumberColumn(columnId); - } - - public static boolean isShardRowIdColumn(long columnId) - { - return columnId == SHARD_ROW_ID_COLUMN_ID; - } - - public static boolean isShardUuidColumn(long columnId) - { - return columnId == SHARD_UUID_COLUMN_ID; - } - - public static RaptorColumnHandle shardUuidColumnHandle() - { - return new RaptorColumnHandle(SHARD_UUID_COLUMN_NAME, SHARD_UUID_COLUMN_ID, SHARD_UUID_COLUMN_TYPE); - } - - public static boolean isBucketNumberColumn(long columnId) - { - return columnId == BUCKET_NUMBER_COLUMN_ID; - } - - public static RaptorColumnHandle bucketNumberColumnHandle() - { - return new RaptorColumnHandle(BUCKET_NUMBER_COLUMN_NAME, BUCKET_NUMBER_COLUMN_ID, INTEGER); - } - - public static RaptorColumnHandle mergeRowIdHandle() - { - return new RaptorColumnHandle(MERGE_ROW_ID_COLUMN_NAME, MERGE_ROW_ID_COLUMN_ID, MERGE_ROW_ID_COLUMN_TYPE); - } - - public static boolean isMergeRowIdColumn(long columnId) - { - return columnId == MERGE_ROW_ID_COLUMN_ID; - } - - public static boolean isHiddenColumn(long columnId) - { - return columnId < 0; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnector.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnector.java deleted file mode 100644 index 8e362eca5fa0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnector.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.SetMultimap; -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.spi.NodeManager; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorAccessControl; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorNodePartitioningProvider; -import io.trino.spi.connector.ConnectorPageSinkProvider; -import io.trino.spi.connector.ConnectorPageSourceProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.session.PropertyMetadata; -import io.trino.spi.transaction.IsolationLevel; -import jakarta.annotation.PostConstruct; -import org.jdbi.v3.core.Jdbi; - -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ScheduledExecutorService; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Verify.verify; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; -import static io.trino.spi.transaction.IsolationLevel.checkConnectorSupports; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class RaptorConnector - implements Connector -{ - private static final Logger log = Logger.get(RaptorConnector.class); - - private final LifeCycleManager lifeCycleManager; - private final RaptorMetadataFactory metadataFactory; - private final RaptorSplitManager splitManager; - private final RaptorPageSourceProvider pageSourceProvider; - private final RaptorPageSinkProvider pageSinkProvider; - private final RaptorNodePartitioningProvider nodePartitioningProvider; - private final List> sessionProperties; - private final List> tableProperties; - private final Set systemTables; - private final MetadataDao dao; - private final Optional accessControl; - private final boolean coordinator; - - private final ConcurrentMap transactions = new ConcurrentHashMap<>(); - - private final ScheduledExecutorService unblockMaintenanceExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("raptor-unblock-maintenance")); - - @GuardedBy("this") - private final SetMultimap deletions = HashMultimap.create(); - - @Inject - public RaptorConnector( - LifeCycleManager lifeCycleManager, - NodeManager nodeManager, - RaptorMetadataFactory metadataFactory, - RaptorSplitManager splitManager, - RaptorPageSourceProvider pageSourceProvider, - RaptorPageSinkProvider pageSinkProvider, - RaptorNodePartitioningProvider nodePartitioningProvider, - RaptorSessionProperties sessionProperties, - RaptorTableProperties tableProperties, - Set systemTables, - Optional accessControl, - @ForMetadata Jdbi dbi) - { - this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); - this.metadataFactory = requireNonNull(metadataFactory, "metadataFactory is null"); - this.splitManager = requireNonNull(splitManager, "splitManager is null"); - this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); - this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null"); - this.nodePartitioningProvider = requireNonNull(nodePartitioningProvider, "nodePartitioningProvider is null"); - this.sessionProperties = sessionProperties.getSessionProperties(); - this.tableProperties = tableProperties.getTableProperties(); - this.systemTables = requireNonNull(systemTables, "systemTables is null"); - this.accessControl = requireNonNull(accessControl, "accessControl is null"); - this.dao = onDemandDao(dbi, MetadataDao.class); - this.coordinator = nodeManager.getCurrentNode().isCoordinator(); - } - - @PostConstruct - public void start() - { - if (coordinator) { - dao.unblockAllMaintenance(); - } - } - - @Override - public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommit) - { - checkConnectorSupports(READ_COMMITTED, isolationLevel); - RaptorTransactionHandle transaction = new RaptorTransactionHandle(); - transactions.put(transaction, metadataFactory.create(tableId -> beginDelete(tableId, transaction.getUuid()))); - return transaction; - } - - @Override - public void commit(ConnectorTransactionHandle transaction) - { - checkArgument(transactions.remove(transaction) != null, "no such transaction: %s", transaction); - finishDelete(((RaptorTransactionHandle) transaction).getUuid()); - } - - @Override - public void rollback(ConnectorTransactionHandle transaction) - { - RaptorMetadata metadata = transactions.remove(transaction); - checkArgument(metadata != null, "no such transaction: %s", transaction); - finishDelete(((RaptorTransactionHandle) transaction).getUuid()); - metadata.rollback(); - } - - @Override - public ConnectorPageSourceProvider getPageSourceProvider() - { - return pageSourceProvider; - } - - @Override - public ConnectorPageSinkProvider getPageSinkProvider() - { - return pageSinkProvider; - } - - @Override - public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transaction) - { - RaptorMetadata metadata = transactions.get(transaction); - checkArgument(metadata != null, "no such transaction: %s", transaction); - return metadata; - } - - @Override - public ConnectorSplitManager getSplitManager() - { - return splitManager; - } - - @Override - public ConnectorNodePartitioningProvider getNodePartitioningProvider() - { - return nodePartitioningProvider; - } - - @Override - public List> getSessionProperties() - { - return sessionProperties; - } - - @Override - public List> getTableProperties() - { - return tableProperties; - } - - @Override - public Set getSystemTables() - { - return systemTables; - } - - @Override - public ConnectorAccessControl getAccessControl() - { - return accessControl.orElseThrow(UnsupportedOperationException::new); - } - - @Override - public final void shutdown() - { - lifeCycleManager.stop(); - } - - private synchronized void beginDelete(long tableId, UUID transactionId) - { - dao.blockMaintenance(tableId); - verify(deletions.put(tableId, transactionId)); - } - - private synchronized void finishDelete(UUID transactionId) - { - deletions.entries().stream() - .filter(entry -> entry.getValue().equals(transactionId)) - .findFirst() - .ifPresent(entry -> { - long tableId = entry.getKey(); - deletions.remove(tableId, transactionId); - if (!deletions.containsKey(tableId)) { - unblockMaintenance(tableId); - } - }); - } - - private void unblockMaintenance(long tableId) - { - try { - dao.unblockMaintenance(tableId); - } - catch (Throwable t) { - log.warn(t, "Failed to unblock maintenance for table ID %s, will retry", tableId); - unblockMaintenanceExecutor.schedule(() -> unblockMaintenance(tableId), 2, SECONDS); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnectorFactory.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnectorFactory.java deleted file mode 100644 index be1399b59a4b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnectorFactory.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableMap; -import com.google.inject.Injector; -import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; -import io.trino.plugin.base.CatalogNameModule; -import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; -import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.plugin.raptor.legacy.backup.BackupModule; -import io.trino.plugin.raptor.legacy.security.RaptorSecurityModule; -import io.trino.plugin.raptor.legacy.storage.StorageModule; -import io.trino.spi.NodeManager; -import io.trino.spi.PageSorter; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; -import org.weakref.jmx.guice.MBeanModule; - -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static io.trino.plugin.base.Versions.checkStrictSpiVersionMatch; -import static java.util.Objects.requireNonNull; - -public class RaptorConnectorFactory - implements ConnectorFactory -{ - private final String name; - private final Module metadataModule; - private final Map backupProviders; - - public RaptorConnectorFactory(String name, Module metadataModule, Map backupProviders) - { - checkArgument(!isNullOrEmpty(name), "name is null or empty"); - this.name = name; - this.metadataModule = requireNonNull(metadataModule, "metadataModule is null"); - this.backupProviders = ImmutableMap.copyOf(requireNonNull(backupProviders, "backupProviders is null")); - } - - @Override - public String getName() - { - return name; - } - - @Override - public Connector create(String catalogName, Map config, ConnectorContext context) - { - checkStrictSpiVersionMatch(context, this); - - Bootstrap app = new Bootstrap( - new CatalogNameModule(catalogName), - new JsonModule(), - new MBeanModule(), - new ConnectorObjectNameGeneratorModule("io.trino.plugin.raptor.legacy", "trino.plugin.raptor.legacy"), - new MBeanServerModule(), - binder -> { - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(PageSorter.class).toInstance(context.getPageSorter()); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }, - metadataModule, - new BackupModule(backupProviders), - new StorageModule(), - new RaptorModule(), - new RaptorSecurityModule()); - - Injector injector = app - .doNotInitializeLogging() - .setRequiredConfigurationProperties(config) - .initialize(); - - return injector.getInstance(RaptorConnector.class); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorErrorCode.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorErrorCode.java deleted file mode 100644 index e16ba20ef687..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorErrorCode.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.spi.ErrorCode; -import io.trino.spi.ErrorCodeSupplier; -import io.trino.spi.ErrorType; - -import static io.trino.spi.ErrorType.EXTERNAL; - -public enum RaptorErrorCode - implements ErrorCodeSupplier -{ - RAPTOR_ERROR(0, EXTERNAL), - RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS(1, EXTERNAL), - RAPTOR_NO_HOST_FOR_SHARD(2, EXTERNAL), - RAPTOR_RECOVERY_ERROR(3, EXTERNAL), - RAPTOR_BACKUP_TIMEOUT(4, EXTERNAL), - RAPTOR_METADATA_ERROR(5, EXTERNAL), - RAPTOR_BACKUP_ERROR(6, EXTERNAL), - RAPTOR_BACKUP_NOT_FOUND(7, EXTERNAL), - RAPTOR_REASSIGNMENT_DELAY(8, EXTERNAL), - RAPTOR_REASSIGNMENT_THROTTLE(9, EXTERNAL), - RAPTOR_RECOVERY_TIMEOUT(10, EXTERNAL), - RAPTOR_CORRUPT_METADATA(11, EXTERNAL), - RAPTOR_LOCAL_DISK_FULL(12, EXTERNAL), - RAPTOR_BACKUP_CORRUPTION(13, EXTERNAL); - - private final ErrorCode errorCode; - - RaptorErrorCode(int code, ErrorType type) - { - errorCode = new ErrorCode(code + 0x0300_0000, name(), type); - } - - @Override - public ErrorCode toErrorCode() - { - return errorCode; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorInsertTableHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorInsertTableHandle.java deleted file mode 100644 index 551e7c4647f7..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorInsertTableHandle.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class RaptorInsertTableHandle - implements ConnectorInsertTableHandle -{ - private final long transactionId; - private final long tableId; - private final List columnHandles; - private final List columnTypes; - private final Optional externalBatchId; - private final List sortColumnHandles; - private final List sortOrders; - private final OptionalInt bucketCount; - private final List bucketColumnHandles; - private final Optional temporalColumnHandle; - - @JsonCreator - public RaptorInsertTableHandle( - @JsonProperty("transactionId") long transactionId, - @JsonProperty("tableId") long tableId, - @JsonProperty("columnHandles") List columnHandles, - @JsonProperty("columnTypes") List columnTypes, - @JsonProperty("externalBatchId") Optional externalBatchId, - @JsonProperty("sortColumnHandles") List sortColumnHandles, - @JsonProperty("sortOrders") List sortOrders, - @JsonProperty("bucketCount") OptionalInt bucketCount, - @JsonProperty("bucketColumnHandles") List bucketColumnHandles, - @JsonProperty("temporalColumnHandle") Optional temporalColumnHandle) - { - checkArgument(tableId > 0, "tableId must be greater than zero"); - - this.transactionId = transactionId; - this.tableId = tableId; - this.columnHandles = ImmutableList.copyOf(requireNonNull(columnHandles, "columnHandles is null")); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.externalBatchId = requireNonNull(externalBatchId, "externalBatchId is null"); - - this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); - this.sortColumnHandles = ImmutableList.copyOf(requireNonNull(sortColumnHandles, "sortColumnHandles is null")); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.bucketColumnHandles = ImmutableList.copyOf(requireNonNull(bucketColumnHandles, "bucketColumnHandles is null")); - this.temporalColumnHandle = requireNonNull(temporalColumnHandle, "temporalColumnHandle is null"); - } - - @JsonProperty - public long getTransactionId() - { - return transactionId; - } - - @JsonProperty - public long getTableId() - { - return tableId; - } - - @JsonProperty - public List getColumnHandles() - { - return columnHandles; - } - - @JsonProperty - public List getColumnTypes() - { - return columnTypes; - } - - @JsonProperty - public Optional getExternalBatchId() - { - return externalBatchId; - } - - @JsonProperty - public List getSortColumnHandles() - { - return sortColumnHandles; - } - - @JsonProperty - public List getSortOrders() - { - return sortOrders; - } - - @JsonProperty - public OptionalInt getBucketCount() - { - return bucketCount; - } - - @JsonProperty - public List getBucketColumnHandles() - { - return bucketColumnHandles; - } - - @JsonProperty - public Optional getTemporalColumnHandle() - { - return temporalColumnHandle; - } - - @Override - public String toString() - { - return String.valueOf(tableId); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeSink.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeSink.java deleted file mode 100644 index 9f1ce30ae412..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeSink.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.trino.plugin.raptor.legacy.metadata.ShardDelta; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.ShardRewriter; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ConnectorMergeSink; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.MergePage; -import io.trino.spi.type.UuidType; - -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.OptionalInt; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; - -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.trino.spi.block.RowBlock.getRowFieldsFromBlock; -import static io.trino.spi.connector.MergePage.createDeleteAndInsertPages; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.UuidType.trinoUuidToJavaUuid; -import static java.lang.Math.toIntExact; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.allOf; -import static java.util.stream.Collectors.toUnmodifiableList; - -public class RaptorMergeSink - implements ConnectorMergeSink -{ - private static final JsonCodec SHARD_INFO_CODEC = jsonCodec(ShardInfo.class); - private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); - - private final ConnectorPageSink pageSink; - private final StorageManager storageManager; - private final long transactionId; - private final int columnCount; - private final Map> rowsToDelete = new HashMap<>(); - - public RaptorMergeSink(ConnectorPageSink pageSink, StorageManager storageManager, long transactionId, int columnCount) - { - this.pageSink = requireNonNull(pageSink, "pageSink is null"); - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - this.transactionId = transactionId; - this.columnCount = columnCount; - } - - @Override - public void storeMergedRows(Page page) - { - MergePage mergePage = createDeleteAndInsertPages(page, columnCount); - - mergePage.getInsertionsPage().ifPresent(pageSink::appendPage); - - mergePage.getDeletionsPage().ifPresent(deletions -> { - List fields = getRowFieldsFromBlock(deletions.getBlock(deletions.getChannelCount() - 1)); - Block shardBucketBlock = fields.get(0); - Block shardUuidBlock = fields.get(1); - Block shardRowIdBlock = fields.get(2); - - for (int position = 0; position < shardRowIdBlock.getPositionCount(); position++) { - OptionalInt bucketNumber = shardBucketBlock.isNull(position) - ? OptionalInt.empty() - : OptionalInt.of(INTEGER.getInt(shardBucketBlock, position)); - UUID uuid = trinoUuidToJavaUuid(UuidType.UUID.getSlice(shardUuidBlock, position)); - int rowId = toIntExact(BIGINT.getLong(shardRowIdBlock, position)); - Entry entry = rowsToDelete.computeIfAbsent(uuid, _ -> Map.entry(bucketNumber, new BitSet())); - verify(entry.getKey().equals(bucketNumber), "multiple bucket numbers for same shard"); - entry.getValue().set(rowId); - } - }); - } - - @Override - public CompletableFuture> finish() - { - List>> futures = new ArrayList<>(); - - rowsToDelete.forEach((uuid, entry) -> { - OptionalInt bucketNumber = entry.getKey(); - BitSet rowIds = entry.getValue(); - ShardRewriter rewriter = storageManager.createShardRewriter(transactionId, bucketNumber, uuid); - futures.add(rewriter.rewrite(rowIds)); - }); - - futures.add(pageSink.finish().thenApply(slices -> { - List newShards = slices.stream() - .map(slice -> SHARD_INFO_CODEC.fromJson(slice.getBytes())) - .collect(toImmutableList()); - ShardDelta delta = new ShardDelta(ImmutableList.of(), newShards); - return ImmutableList.of(Slices.wrappedBuffer(SHARD_DELTA_CODEC.toJsonBytes(delta))); - })); - - return allOf(futures.toArray(CompletableFuture[]::new)) - .thenApply(_ -> futures.stream() - .map(CompletableFuture::join) - .flatMap(Collection::stream) - .collect(toUnmodifiableList())); - } - - @Override - public void abort() - { - pageSink.abort(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeTableHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeTableHandle.java deleted file mode 100644 index 4240bd5c0e67..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeTableHandle.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.connector.ConnectorMergeTableHandle; - -import static java.util.Objects.requireNonNull; - -public class RaptorMergeTableHandle - implements ConnectorMergeTableHandle -{ - private final RaptorTableHandle tableHandle; - private final RaptorInsertTableHandle insertTableHandle; - - @JsonCreator - public RaptorMergeTableHandle( - @JsonProperty RaptorTableHandle tableHandle, - @JsonProperty RaptorInsertTableHandle insertTableHandle) - { - this.tableHandle = requireNonNull(tableHandle, "tableHandle is null"); - this.insertTableHandle = requireNonNull(insertTableHandle, "insertTableHandle is null"); - } - - @Override - @JsonProperty - public RaptorTableHandle getTableHandle() - { - return tableHandle; - } - - @JsonProperty - public RaptorInsertTableHandle getInsertTableHandle() - { - return insertTableHandle; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadata.java deleted file mode 100644 index 31fecf590e72..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadata.java +++ /dev/null @@ -1,1033 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimaps; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; -import io.airlift.log.Logger; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardDelta; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.plugin.raptor.legacy.metadata.ViewResult; -import io.trino.plugin.raptor.legacy.systemtables.ColumnRangesSystemTable; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorMergeTableHandle; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorOutputMetadata; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorPartitioningHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableLayout; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTablePartitioning; -import io.trino.spi.connector.ConnectorTableProperties; -import io.trino.spi.connector.ConnectorTableVersion; -import io.trino.spi.connector.ConnectorViewDefinition; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.ConstraintApplicationResult; -import io.trino.spi.connector.RetryMode; -import io.trino.spi.connector.RowChangeParadigm; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.connector.TableNotFoundException; -import io.trino.spi.connector.ViewNotFoundException; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.statistics.ComputedStatistics; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Jdbi; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.LongConsumer; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.collect.MoreCollectors.toOptional; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.trino.plugin.raptor.legacy.RaptorBucketFunction.validateBucketType; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.BUCKET_NUMBER_COLUMN_NAME; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_NAME; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.bucketNumberColumnHandle; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isHiddenColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.mergeRowIdHandle; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.shardUuidColumnHandle; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getExternalBatchId; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getOneSplitPerBucketThreshold; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.BUCKETED_ON_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.BUCKET_COUNT_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.DISTRIBUTION_NAME_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.ORDERING_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.ORGANIZED_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getBucketColumns; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getBucketCount; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getDistributionName; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getSortColumns; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getTemporalColumn; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.isOrganized; -import static io.trino.plugin.raptor.legacy.systemtables.ColumnRangesSystemTable.getSourceTable; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.daoTransaction; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runIgnoringConstraintViolation; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runTransaction; -import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; -import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; -import static io.trino.spi.StandardErrorCode.NOT_FOUND; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.connector.RetryMode.NO_RETRIES; -import static io.trino.spi.connector.RowChangeParadigm.DELETE_ROW_AND_INSERT_ROW; -import static io.trino.spi.connector.SaveMode.REPLACE; -import static io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static java.lang.String.format; -import static java.util.Collections.nCopies; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toCollection; -import static java.util.stream.Collectors.toList; - -public class RaptorMetadata - implements ConnectorMetadata -{ - private static final Logger log = Logger.get(RaptorMetadata.class); - - private static final JsonCodec SHARD_INFO_CODEC = jsonCodec(ShardInfo.class); - private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); - - private static final JsonCodec VIEW_CODEC = - new JsonCodecFactory(new ObjectMapperProvider()).jsonCodec(ConnectorViewDefinition.class); - - private final Jdbi dbi; - private final MetadataDao dao; - private final ShardManager shardManager; - private final LongConsumer beginDeleteForTableId; - - private final AtomicReference currentTransactionId = new AtomicReference<>(); - - public RaptorMetadata(Jdbi dbi, ShardManager shardManager) - { - this(dbi, shardManager, tableId -> {}); - } - - public RaptorMetadata(Jdbi dbi, ShardManager shardManager, LongConsumer beginDeleteForTableId) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.dao = onDemandDao(dbi, MetadataDao.class); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.beginDeleteForTableId = requireNonNull(beginDeleteForTableId, "beginDeleteForTableId is null"); - } - - @Override - public List listSchemaNames(ConnectorSession session) - { - return dao.listSchemaNames(); - } - - @Override - public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName, Optional startVersion, Optional endVersion) - { - if (startVersion.isPresent() || endVersion.isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support versioned tables"); - } - - return getTableHandle(tableName); - } - - private RaptorTableHandle getTableHandle(SchemaTableName tableName) - { - requireNonNull(tableName, "tableName is null"); - Table table = dao.getTableInformation(tableName.getSchemaName(), tableName.getTableName()); - if (table == null) { - return null; - } - List tableColumns = dao.listTableColumns(table.getTableId()); - checkArgument(!tableColumns.isEmpty(), "Table '%s' does not have any columns", tableName); - - return new RaptorTableHandle( - tableName.getSchemaName(), - tableName.getTableName(), - table.getTableId(), - table.getDistributionId(), - table.getDistributionName(), - table.getBucketCount(), - table.isOrganized(), - TupleDomain.all(), - table.getDistributionId().map(shardManager::getBucketAssignments)); - } - - @Override - public Optional getSystemTable(ConnectorSession session, SchemaTableName tableName) - { - return getSourceTable(tableName) - .map(this::getTableHandle) - .map(handle -> new ColumnRangesSystemTable(handle, dbi)); - } - - @Override - public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) - { - RaptorTableHandle handle = (RaptorTableHandle) tableHandle; - SchemaTableName tableName = new SchemaTableName(handle.getSchemaName(), handle.getTableName()); - List tableColumns = dao.listTableColumns(handle.getTableId()); - if (tableColumns.isEmpty()) { - throw new TableNotFoundException(tableName); - } - - ImmutableMap.Builder properties = ImmutableMap.builder(); - SortedMap bucketing = new TreeMap<>(); - SortedMap ordering = new TreeMap<>(); - - for (TableColumn column : tableColumns) { - if (column.isTemporal()) { - properties.put(TEMPORAL_COLUMN_PROPERTY, column.getColumnName()); - } - column.getBucketOrdinal().ifPresent(bucketOrdinal -> bucketing.put(bucketOrdinal, column.getColumnName())); - column.getSortOrdinal().ifPresent(sortOrdinal -> ordering.put(sortOrdinal, column.getColumnName())); - } - - if (!bucketing.isEmpty()) { - properties.put(BUCKETED_ON_PROPERTY, ImmutableList.copyOf(bucketing.values())); - } - if (!ordering.isEmpty()) { - properties.put(ORDERING_PROPERTY, ImmutableList.copyOf(ordering.values())); - } - - handle.getBucketCount().ifPresent(bucketCount -> properties.put(BUCKET_COUNT_PROPERTY, bucketCount)); - handle.getDistributionName().ifPresent(distributionName -> properties.put(DISTRIBUTION_NAME_PROPERTY, distributionName)); - // Only display organization property if set - if (handle.isOrganized()) { - properties.put(ORGANIZED_PROPERTY, true); - } - - List columns = tableColumns.stream() - .map(TableColumn::toColumnMetadata) - .collect(toCollection(ArrayList::new)); - - columns.add(hiddenColumn(SHARD_UUID_COLUMN_NAME, SHARD_UUID_COLUMN_TYPE)); - - if (handle.isBucketed()) { - columns.add(hiddenColumn(BUCKET_NUMBER_COLUMN_NAME, INTEGER)); - } - - return new ConnectorTableMetadata(tableName, columns, properties.buildOrThrow()); - } - - @Override - public List listTables(ConnectorSession session, Optional schemaName) - { - // Deduplicate with set because state may change concurrently - return ImmutableSet.builder() - .addAll(dao.listTables(schemaName.orElse(null))) - .addAll(listViews(session, schemaName)) - .build().asList(); - } - - @Override - public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) - { - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (TableColumn tableColumn : dao.listTableColumns(raptorTableHandle.getTableId())) { - builder.put(tableColumn.getColumnName(), getRaptorColumnHandle(tableColumn)); - } - - RaptorColumnHandle uuidColumn = shardUuidColumnHandle(); - builder.put(uuidColumn.getColumnName(), uuidColumn); - - if (raptorTableHandle.isBucketed()) { - RaptorColumnHandle bucketNumberColumn = bucketNumberColumnHandle(); - builder.put(bucketNumberColumn.getColumnName(), bucketNumberColumn); - } - - return builder.buildOrThrow(); - } - - @Override - public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) - { - RaptorColumnHandle column = (RaptorColumnHandle) columnHandle; - - if (isHiddenColumn(column.getColumnId())) { - return hiddenColumn(column.getColumnName(), column.getColumnType()); - } - - return new ColumnMetadata(column.getColumnName(), column.getColumnType()); - } - - @Override - public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) - { - requireNonNull(prefix, "prefix is null"); - - ImmutableListMultimap.Builder columns = ImmutableListMultimap.builder(); - for (TableColumn tableColumn : dao.listTableColumns(prefix.getSchema().orElse(null), prefix.getTable().orElse(null))) { - ColumnMetadata columnMetadata = new ColumnMetadata(tableColumn.getColumnName(), tableColumn.getDataType()); - columns.put(tableColumn.getTable(), columnMetadata); - } - return Multimaps.asMap(columns.build()); - } - - @Override - public Optional> applyFilter(ConnectorSession session, ConnectorTableHandle handle, Constraint constraint) - { - RaptorTableHandle table = (RaptorTableHandle) handle; - TupleDomain newDomain = constraint.getSummary().transformKeys(RaptorColumnHandle.class::cast); - - if (newDomain.equals(table.getConstraint())) { - return Optional.empty(); - } - - return Optional.of(new ConstraintApplicationResult<>( - new RaptorTableHandle(table.getSchemaName(), - table.getTableName(), - table.getTableId(), - table.getDistributionId(), - table.getDistributionName(), - table.getBucketCount(), - table.isOrganized(), - newDomain.intersect(table.getConstraint()), - table.getBucketAssignments()), - constraint.getSummary(), - constraint.getExpression(), - false)); - } - - @Override - public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle handle) - { - RaptorTableHandle table = (RaptorTableHandle) handle; - - if (table.getPartitioningHandle().isEmpty()) { - return new ConnectorTableProperties(); - } - - List bucketColumnHandles = getBucketColumnHandles(table.getTableId()); - - RaptorPartitioningHandle partitioning = table.getPartitioningHandle().get(); - - boolean oneSplitPerBucket = table.getBucketCount().getAsInt() >= getOneSplitPerBucketThreshold(session); - - return new ConnectorTableProperties( - TupleDomain.all(), - Optional.of(new ConnectorTablePartitioning( - partitioning, - ImmutableList.copyOf(bucketColumnHandles), - oneSplitPerBucket)), - Optional.empty(), - ImmutableList.of()); - } - - @Override - public Optional getNewTableLayout(ConnectorSession session, ConnectorTableMetadata metadata) - { - ImmutableMap.Builder map = ImmutableMap.builder(); - long columnId = 1; - for (ColumnMetadata column : metadata.getColumns()) { - map.put(column.getName(), new RaptorColumnHandle(column.getName(), columnId, column.getType())); - columnId++; - } - - Optional distribution = getOrCreateDistribution(map.buildOrThrow(), metadata.getProperties()); - if (distribution.isEmpty()) { - return Optional.empty(); - } - - List partitionColumns = distribution.get().getBucketColumns().stream() - .map(RaptorColumnHandle::getColumnName) - .collect(toList()); - - long distributionId = distribution.get().getDistributionId(); - List bucketAssignments = shardManager.getBucketAssignments(distributionId); - ConnectorPartitioningHandle partitioning = new RaptorPartitioningHandle(distributionId, bucketAssignments); - - return Optional.of(new ConnectorTableLayout(partitioning, partitionColumns, false)); - } - - private Optional getOrCreateDistribution(Map columnHandleMap, Map properties) - { - OptionalInt bucketCount = getBucketCount(properties); - List bucketColumnHandles = getBucketColumnHandles(getBucketColumns(properties), columnHandleMap); - - if (bucketCount.isPresent() && bucketColumnHandles.isEmpty()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKETED_ON_PROPERTY, BUCKET_COUNT_PROPERTY)); - } - if (bucketCount.isEmpty() && !bucketColumnHandles.isEmpty()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKET_COUNT_PROPERTY, BUCKETED_ON_PROPERTY)); - } - ImmutableList.Builder bucketColumnTypes = ImmutableList.builder(); - for (RaptorColumnHandle column : bucketColumnHandles) { - validateBucketType(column.getColumnType()); - bucketColumnTypes.add(column.getColumnType()); - } - - long distributionId; - String distributionName = getDistributionName(properties); - if (distributionName != null) { - if (bucketColumnHandles.isEmpty()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKETED_ON_PROPERTY, DISTRIBUTION_NAME_PROPERTY)); - } - - Distribution distribution = dao.getDistribution(distributionName); - if (distribution == null) { - if (bucketCount.isEmpty()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Distribution does not exist and bucket count is not specified"); - } - distribution = getOrCreateDistribution(distributionName, bucketColumnTypes.build(), bucketCount.getAsInt()); - } - distributionId = distribution.getId(); - - if (bucketCount.isPresent() && (distribution.getBucketCount() != bucketCount.getAsInt())) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Bucket count must match distribution"); - } - if (!distribution.getColumnTypes().equals(bucketColumnTypes.build())) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Bucket column types must match distribution"); - } - } - else if (bucketCount.isPresent()) { - String types = Distribution.serializeColumnTypes(bucketColumnTypes.build()); - distributionId = dao.insertDistribution(null, types, bucketCount.getAsInt()); - } - else { - return Optional.empty(); - } - - shardManager.createBuckets(distributionId, bucketCount.getAsInt()); - - return Optional.of(new DistributionInfo(distributionId, bucketCount.getAsInt(), bucketColumnHandles)); - } - - private Distribution getOrCreateDistribution(String name, List columnTypes, int bucketCount) - { - String types = Distribution.serializeColumnTypes(columnTypes); - runIgnoringConstraintViolation(() -> dao.insertDistribution(name, types, bucketCount)); - - Distribution distribution = dao.getDistribution(name); - if (distribution == null) { - throw new TrinoException(RAPTOR_ERROR, "Distribution does not exist after insert"); - } - return distribution; - } - - @Override - public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, SaveMode saveMode) - { - if (saveMode == REPLACE) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support replacing tables"); - } - Optional layout = getNewTableLayout(session, tableMetadata); - finishCreateTable(session, beginCreateTable(session, tableMetadata, layout, NO_RETRIES, false), ImmutableList.of(), ImmutableList.of()); - } - - @Override - public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) - { - RaptorTableHandle raptorHandle = (RaptorTableHandle) tableHandle; - shardManager.dropTable(raptorHandle.getTableId()); - } - - @Override - public void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle, SchemaTableName newTableName) - { - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - runTransaction(dbi, handle -> { - MetadataDao dao = handle.attach(MetadataDao.class); - dao.renameTable(table.getTableId(), newTableName.getSchemaName(), newTableName.getTableName()); - return null; - }); - } - - @Override - public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) - { - if (column.getComment() != null) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support adding columns with comments"); - } - - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - - // Always add new columns to the end. - List existingColumns = dao.listTableColumns(table.getSchemaName(), table.getTableName()); - TableColumn lastColumn = existingColumns.getLast(); - long columnId = lastColumn.getColumnId() + 1; - int ordinalPosition = lastColumn.getOrdinalPosition() + 1; - - String type = column.getType().getTypeId().getId(); - daoTransaction(dbi, MetadataDao.class, dao -> { - dao.insertColumn(table.getTableId(), columnId, column.getName(), ordinalPosition, type, null, null); - dao.updateTableVersion(table.getTableId(), session.getStart().toEpochMilli()); - }); - - shardManager.addColumn(table.getTableId(), new ColumnInfo(columnId, column.getType())); - } - - @Override - public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) - { - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - RaptorColumnHandle sourceColumn = (RaptorColumnHandle) source; - daoTransaction(dbi, MetadataDao.class, dao -> { - dao.renameColumn(table.getTableId(), sourceColumn.getColumnId(), target); - dao.updateTableVersion(table.getTableId(), session.getStart().toEpochMilli()); - }); - } - - @Override - public void dropColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle column) - { - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - RaptorColumnHandle raptorColumn = (RaptorColumnHandle) column; - - List existingColumns = dao.listTableColumns(table.getSchemaName(), table.getTableName()); - if (existingColumns.size() <= 1) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop the only column in a table"); - } - long maxColumnId = existingColumns.stream().mapToLong(TableColumn::getColumnId).max().getAsLong(); - if (raptorColumn.getColumnId() == maxColumnId) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop the column which has the largest column ID in the table"); - } - - if (getBucketColumnHandles(table.getTableId()).contains(column)) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop bucket columns"); - } - - Optional.ofNullable(dao.getTemporalColumnId(table.getTableId())).ifPresent(tempColumnId -> { - if (raptorColumn.getColumnId() == tempColumnId) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop the temporal column"); - } - }); - - if (getSortColumnHandles(table.getTableId()).contains(raptorColumn)) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop sort columns"); - } - - daoTransaction(dbi, MetadataDao.class, dao -> { - dao.dropColumn(table.getTableId(), raptorColumn.getColumnId()); - dao.updateTableVersion(table.getTableId(), session.getStart().toEpochMilli()); - }); - - // TODO: drop column from index table - } - - @Override - public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout, RetryMode retryMode, boolean replace) - { - if (retryMode != NO_RETRIES) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support query retries"); - } - if (replace) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support replacing tables"); - } - if (tableMetadata.getComment().isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support creating tables with table comment"); - } - - if (viewExists(session, tableMetadata.getTable())) { - throw new TrinoException(ALREADY_EXISTS, "View already exists: " + tableMetadata.getTable()); - } - - Optional partitioning = layout - .map(ConnectorTableLayout::getPartitioning) - .map(Optional::get) - .map(RaptorPartitioningHandle.class::cast); - - ImmutableList.Builder columnHandles = ImmutableList.builder(); - ImmutableList.Builder columnTypes = ImmutableList.builder(); - - long columnId = 1; - for (ColumnMetadata column : tableMetadata.getColumns()) { - if (column.getComment() != null) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support creating tables with column comment"); - } - columnHandles.add(new RaptorColumnHandle(column.getName(), columnId, column.getType())); - columnTypes.add(column.getType()); - columnId++; - } - Map columnHandleMap = Maps.uniqueIndex(columnHandles.build(), RaptorColumnHandle::getColumnName); - - List sortColumnHandles = getSortColumnHandles(getSortColumns(tableMetadata.getProperties()), columnHandleMap); - Optional temporalColumnHandle = getTemporalColumnHandle(getTemporalColumn(tableMetadata.getProperties()), columnHandleMap); - - if (temporalColumnHandle.isPresent()) { - RaptorColumnHandle column = temporalColumnHandle.get(); - if (!column.getColumnType().equals(TIMESTAMP_MILLIS) && !column.getColumnType().equals(DATE)) { - throw new TrinoException(NOT_SUPPORTED, "Temporal column must be of type timestamp or date: " + column.getColumnName()); - } - } - - boolean organized = isOrganized(tableMetadata.getProperties()); - if (organized) { - if (temporalColumnHandle.isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "Table with temporal columns cannot be organized"); - } - if (sortColumnHandles.isEmpty()) { - throw new TrinoException(NOT_SUPPORTED, "Table organization requires an ordering"); - } - } - - long transactionId = shardManager.beginTransaction(); - - setTransactionId(transactionId); - - Optional distribution = partitioning.map(handle -> - getDistributionInfo(handle.getDistributionId(), columnHandleMap, tableMetadata.getProperties())); - - return new RaptorOutputTableHandle( - transactionId, - tableMetadata.getTable().getSchemaName(), - tableMetadata.getTable().getTableName(), - columnHandles.build(), - columnTypes.build(), - sortColumnHandles, - nCopies(sortColumnHandles.size(), ASC_NULLS_FIRST), - temporalColumnHandle, - distribution.map(info -> OptionalLong.of(info.getDistributionId())).orElse(OptionalLong.empty()), - distribution.map(info -> OptionalInt.of(info.getBucketCount())).orElse(OptionalInt.empty()), - organized, - distribution.map(DistributionInfo::getBucketColumns).orElse(ImmutableList.of())); - } - - private DistributionInfo getDistributionInfo(long distributionId, Map columnHandleMap, Map properties) - { - Distribution distribution = dao.getDistribution(distributionId); - if (distribution == null) { - throw new TrinoException(RAPTOR_ERROR, "Distribution ID does not exist: " + distributionId); - } - List bucketColumnHandles = getBucketColumnHandles(getBucketColumns(properties), columnHandleMap); - return new DistributionInfo(distributionId, distribution.getBucketCount(), bucketColumnHandles); - } - - private static Optional getTemporalColumnHandle(String temporalColumn, Map columnHandleMap) - { - if (temporalColumn == null) { - return Optional.empty(); - } - - RaptorColumnHandle handle = columnHandleMap.get(temporalColumn); - if (handle == null) { - throw new TrinoException(NOT_FOUND, "Temporal column does not exist: " + temporalColumn); - } - return Optional.of(handle); - } - - private static List getSortColumnHandles(List sortColumns, Map columnHandleMap) - { - ImmutableList.Builder columnHandles = ImmutableList.builder(); - for (String column : sortColumns) { - if (!columnHandleMap.containsKey(column)) { - throw new TrinoException(NOT_FOUND, "Ordering column does not exist: " + column); - } - columnHandles.add(columnHandleMap.get(column)); - } - return columnHandles.build(); - } - - private static List getBucketColumnHandles(List bucketColumns, Map columnHandleMap) - { - ImmutableList.Builder columnHandles = ImmutableList.builder(); - for (String column : bucketColumns) { - if (!columnHandleMap.containsKey(column)) { - throw new TrinoException(NOT_FOUND, "Bucketing column does not exist: " + column); - } - columnHandles.add(columnHandleMap.get(column)); - } - return columnHandles.build(); - } - - @Override - public Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle outputTableHandle, Collection fragments, Collection computedStatistics) - { - RaptorOutputTableHandle table = (RaptorOutputTableHandle) outputTableHandle; - long transactionId = table.getTransactionId(); - long updateTime = session.getStart().toEpochMilli(); - - long newTableId = runTransaction(dbi, dbiHandle -> { - MetadataDao dao = dbiHandle.attach(MetadataDao.class); - - Long distributionId = table.getDistributionId().isPresent() ? table.getDistributionId().getAsLong() : null; - // TODO: update default value of organization_enabled to true - long tableId = dao.insertTable(table.getSchemaName(), table.getTableName(), true, table.isOrganized(), distributionId, updateTime); - - List sortColumnHandles = table.getSortColumnHandles(); - List bucketColumnHandles = table.getBucketColumnHandles(); - - for (int i = 0; i < table.getColumnTypes().size(); i++) { - RaptorColumnHandle column = table.getColumnHandles().get(i); - - int columnId = i + 1; - String type = table.getColumnTypes().get(i).getTypeId().getId(); - Integer sortPosition = sortColumnHandles.contains(column) ? sortColumnHandles.indexOf(column) : null; - Integer bucketPosition = bucketColumnHandles.contains(column) ? bucketColumnHandles.indexOf(column) : null; - - dao.insertColumn(tableId, columnId, column.getColumnName(), i, type, sortPosition, bucketPosition); - - if (table.getTemporalColumnHandle().isPresent() && table.getTemporalColumnHandle().get().equals(column)) { - dao.updateTemporalColumnId(tableId, columnId); - } - } - - return tableId; - }); - - List columns = table.getColumnHandles().stream().map(ColumnInfo::fromHandle).collect(toList()); - - OptionalLong temporalColumnId = table.getTemporalColumnHandle().map(RaptorColumnHandle::getColumnId) - .map(OptionalLong::of) - .orElse(OptionalLong.empty()); - - // TODO: refactor this to avoid creating an empty table on failure - shardManager.createTable(newTableId, columns, table.getBucketCount().isPresent(), temporalColumnId); - shardManager.commitShards(transactionId, newTableId, columns, parseFragments(fragments), Optional.empty(), updateTime); - - clearRollback(); - - return Optional.empty(); - } - - @Override - public Optional getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) - { - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - if (table.getPartitioningHandle().isEmpty()) { - return Optional.empty(); - } - return Optional.of(new ConnectorTableLayout( - table.getPartitioningHandle().get(), - getBucketColumnHandles(table.getTableId()).stream() - .map(RaptorColumnHandle::getColumnName) - .collect(toImmutableList()), - false)); - } - - @Override - public RaptorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle, List columns, RetryMode retryMode) - { - if (retryMode != NO_RETRIES) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support query retries"); - } - - RaptorTableHandle handle = (RaptorTableHandle) tableHandle; - long tableId = handle.getTableId(); - - ImmutableList.Builder columnHandlesBuilder = ImmutableList.builder(); - ImmutableList.Builder columnTypes = ImmutableList.builder(); - for (TableColumn column : dao.listTableColumns(tableId)) { - columnHandlesBuilder.add(new RaptorColumnHandle(column.getColumnName(), column.getColumnId(), column.getDataType())); - columnTypes.add(column.getDataType()); - } - - long transactionId = shardManager.beginTransaction(); - - setTransactionId(transactionId); - - Optional externalBatchId = getExternalBatchId(session); - List sortColumnHandles = getSortColumnHandles(tableId); - List bucketColumnHandles = getBucketColumnHandles(tableId); - - ImmutableList columnHandles = columnHandlesBuilder.build(); - Optional temporalColumnHandle = Optional.ofNullable(dao.getTemporalColumnId(tableId)) - .map(temporalColumnId -> getOnlyElement(columnHandles.stream() - .filter(columnHandle -> columnHandle.getColumnId() == temporalColumnId) - .collect(toList()))); - - return new RaptorInsertTableHandle( - transactionId, - tableId, - columnHandles, - columnTypes.build(), - externalBatchId, - sortColumnHandles, - nCopies(sortColumnHandles.size(), ASC_NULLS_FIRST), - handle.getBucketCount(), - bucketColumnHandles, - temporalColumnHandle); - } - - private List getSortColumnHandles(long tableId) - { - return dao.listSortColumns(tableId).stream() - .map(this::getRaptorColumnHandle) - .collect(toList()); - } - - private List getBucketColumnHandles(long tableId) - { - return dao.listBucketColumns(tableId).stream() - .map(this::getRaptorColumnHandle) - .collect(toList()); - } - - @Override - public Optional finishInsert( - ConnectorSession session, - ConnectorInsertTableHandle insertHandle, - List sourceTableHandles, - Collection fragments, - Collection computedStatistics) - { - RaptorInsertTableHandle handle = (RaptorInsertTableHandle) insertHandle; - long transactionId = handle.getTransactionId(); - long tableId = handle.getTableId(); - Optional externalBatchId = handle.getExternalBatchId(); - List columns = handle.getColumnHandles().stream().map(ColumnInfo::fromHandle).collect(toList()); - long updateTime = session.getStart().toEpochMilli(); - - Collection shards = parseFragments(fragments); - log.info("Committing insert into tableId %s (queryId: %s, shards: %s, columns: %s)", handle.getTableId(), session.getQueryId(), shards.size(), columns.size()); - shardManager.commitShards(transactionId, tableId, columns, shards, externalBatchId, updateTime); - - clearRollback(); - - return Optional.empty(); - } - - private void finishDelete(ConnectorSession session, RaptorTableHandle tableHandle, long transactionId, Collection fragments) - { - long tableId = tableHandle.getTableId(); - - List columns = getColumnHandles(session, tableHandle).values().stream() - .map(RaptorColumnHandle.class::cast) - .map(ColumnInfo::fromHandle).collect(toList()); - - Set oldShardUuids = new HashSet<>(); - List newShards = new ArrayList<>(); - - for (Slice fragment : fragments) { - ShardDelta delta = SHARD_DELTA_CODEC.fromJson(fragment.getBytes()); - for (UUID uuid : delta.getOldShardUuids()) { - verify(oldShardUuids.add(uuid), "duplicate old shard: %s", uuid); - } - newShards.addAll(delta.getNewShards()); - } - - OptionalLong updateTime = OptionalLong.of(session.getStart().toEpochMilli()); - - log.info("Finishing update for tableId %s (removed: %s, new: %s)", tableId, oldShardUuids.size(), newShards.size()); - shardManager.replaceShardUuids(transactionId, tableId, columns, oldShardUuids, newShards, updateTime); - - clearRollback(); - } - - @Override - public RowChangeParadigm getRowChangeParadigm(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return DELETE_ROW_AND_INSERT_ROW; - } - - @Override - public ColumnHandle getMergeRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return mergeRowIdHandle(); - } - - @Override - public Optional getUpdateLayout(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return ((RaptorTableHandle) tableHandle).getDistributionId().map(distributionId -> - new RaptorBucketedUpdateHandle(distributionId, shardManager.getBucketAssignments(distributionId))) - .or(() -> Optional.of(RaptorUnbucketedUpdateHandle.INSTANCE)); - } - - @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) - { - RaptorTableHandle handle = (RaptorTableHandle) tableHandle; - - beginDeleteForTableId.accept(handle.getTableId()); - - RaptorInsertTableHandle insertHandle = beginInsert(session, handle, ImmutableList.of(), retryMode); - - return new RaptorMergeTableHandle(handle, insertHandle); - } - - @Override - public void finishMerge(ConnectorSession session, ConnectorMergeTableHandle mergeTableHandle, List sourceTableHandles, Collection fragments, Collection computedStatistics) - { - RaptorMergeTableHandle handle = (RaptorMergeTableHandle) mergeTableHandle; - long transactionId = handle.getInsertTableHandle().getTransactionId(); - finishDelete(session, handle.getTableHandle(), transactionId, fragments); - } - - @Override - public void createView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition definition, Map viewProperties, boolean replace) - { - checkArgument(viewProperties.isEmpty(), "This connector does not support creating views with properties"); - String schemaName = viewName.getSchemaName(); - String tableName = viewName.getTableName(); - String viewData = VIEW_CODEC.toJson(definition); - - if (getTableHandle(viewName) != null) { - throw new TrinoException(ALREADY_EXISTS, "Table already exists: " + viewName); - } - - if (replace) { - daoTransaction(dbi, MetadataDao.class, dao -> { - dao.dropView(schemaName, tableName); - dao.insertView(schemaName, tableName, viewData); - }); - return; - } - - try { - dao.insertView(schemaName, tableName, viewData); - } - catch (TrinoException e) { - if (viewExists(session, viewName)) { - throw new TrinoException(ALREADY_EXISTS, "View already exists: " + viewName); - } - throw e; - } - } - - @Override - public void dropView(ConnectorSession session, SchemaTableName viewName) - { - if (!viewExists(session, viewName)) { - throw new ViewNotFoundException(viewName); - } - dao.dropView(viewName.getSchemaName(), viewName.getTableName()); - } - - @Override - public List listViews(ConnectorSession session, Optional schemaName) - { - return dao.listViews(schemaName.orElse(null)); - } - - @Override - public Map getViews(ConnectorSession session, Optional schemaName) - { - ImmutableMap.Builder map = ImmutableMap.builder(); - for (ViewResult view : dao.getViews(schemaName.orElse(null), null)) { - map.put(view.getName(), VIEW_CODEC.fromJson(view.getData())); - } - return map.buildOrThrow(); - } - - @Override - public Optional getView(ConnectorSession session, SchemaTableName viewName) - { - return dao.getViews(viewName.getSchemaName(), viewName.getTableName()).stream() - .map(view -> VIEW_CODEC.fromJson(view.getData())) - .collect(toOptional()); - } - - private boolean viewExists(ConnectorSession session, SchemaTableName viewName) - { - return getView(session, viewName).isPresent(); - } - - private RaptorColumnHandle getRaptorColumnHandle(TableColumn tableColumn) - { - return new RaptorColumnHandle(tableColumn.getColumnName(), tableColumn.getColumnId(), tableColumn.getDataType()); - } - - private static Collection parseFragments(Collection fragments) - { - return fragments.stream() - .map(fragment -> SHARD_INFO_CODEC.fromJson(fragment.getBytes())) - .collect(toList()); - } - - private static ColumnMetadata hiddenColumn(String name, Type type) - { - return ColumnMetadata.builder() - .setName(name) - .setType(type) - .setHidden(true) - .build(); - } - - private void setTransactionId(long transactionId) - { - checkState(currentTransactionId.compareAndSet(null, transactionId), "current transaction ID already set"); - } - - private void clearRollback() - { - currentTransactionId.set(null); - } - - public void rollback() - { - Long transactionId = currentTransactionId.getAndSet(null); - if (transactionId != null) { - shardManager.rollbackTransaction(transactionId); - } - } - - private static class DistributionInfo - { - private final long distributionId; - private final int bucketCount; - private final List bucketColumns; - - public DistributionInfo(long distributionId, int bucketCount, List bucketColumns) - { - this.distributionId = distributionId; - this.bucketCount = bucketCount; - this.bucketColumns = ImmutableList.copyOf(requireNonNull(bucketColumns, "bucketColumns is null")); - } - - public long getDistributionId() - { - return distributionId; - } - - public int getBucketCount() - { - return bucketCount; - } - - public List getBucketColumns() - { - return bucketColumns; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadataFactory.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadataFactory.java deleted file mode 100644 index 5a5f5865420e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadataFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import org.jdbi.v3.core.Jdbi; - -import java.util.function.LongConsumer; - -import static java.util.Objects.requireNonNull; - -public class RaptorMetadataFactory -{ - private final Jdbi dbi; - private final ShardManager shardManager; - - @Inject - public RaptorMetadataFactory(@ForMetadata Jdbi dbi, ShardManager shardManager) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - } - - public RaptorMetadata create(LongConsumer beginDeleteForTableId) - { - return new RaptorMetadata(dbi, shardManager, beginDeleteForTableId); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorModule.java deleted file mode 100644 index 460cc13fd172..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorModule.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; -import com.google.inject.multibindings.Multibinder; -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.plugin.raptor.legacy.systemtables.ShardMetadataSystemTable; -import io.trino.plugin.raptor.legacy.systemtables.TableMetadataSystemTable; -import io.trino.plugin.raptor.legacy.systemtables.TableStatsSystemTable; -import io.trino.spi.NodeManager; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.type.TypeManager; -import org.jdbi.v3.core.ConnectionFactory; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.sqlobject.SqlObjectPlugin; - -import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; - -public class RaptorModule - implements Module -{ - @Override - public void configure(Binder binder) - { - binder.bind(RaptorConnector.class).in(Scopes.SINGLETON); - binder.bind(RaptorMetadataFactory.class).in(Scopes.SINGLETON); - binder.bind(RaptorSplitManager.class).in(Scopes.SINGLETON); - binder.bind(RaptorPageSourceProvider.class).in(Scopes.SINGLETON); - binder.bind(RaptorPageSinkProvider.class).in(Scopes.SINGLETON); - binder.bind(RaptorNodePartitioningProvider.class).in(Scopes.SINGLETON); - binder.bind(RaptorSessionProperties.class).in(Scopes.SINGLETON); - binder.bind(RaptorTableProperties.class).in(Scopes.SINGLETON); - - Multibinder tableBinder = newSetBinder(binder, SystemTable.class); - tableBinder.addBinding().to(ShardMetadataSystemTable.class).in(Scopes.SINGLETON); - tableBinder.addBinding().to(TableMetadataSystemTable.class).in(Scopes.SINGLETON); - tableBinder.addBinding().to(TableStatsSystemTable.class).in(Scopes.SINGLETON); - } - - @ForMetadata - @Singleton - @Provides - public static Jdbi createJdbi(@ForMetadata ConnectionFactory connectionFactory, TypeManager typeManager) - { - Jdbi dbi = Jdbi.create(connectionFactory) - .installPlugin(new SqlObjectPlugin()) - .registerRowMapper(new TableColumn.Mapper(typeManager)) - .registerRowMapper(new Distribution.Mapper(typeManager)); - createTablesWithRetry(dbi); - return dbi; - } - - @Provides - @Singleton - public static NodeSupplier createNodeSupplier(NodeManager nodeManager) - { - return nodeManager::getWorkerNodes; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorNodePartitioningProvider.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorNodePartitioningProvider.java deleted file mode 100644 index 7b8969efe494..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorNodePartitioningProvider.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.trino.spi.Node; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.connector.ConnectorBucketNodeMap; -import io.trino.spi.connector.ConnectorNodePartitioningProvider; -import io.trino.spi.connector.ConnectorPartitioningHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.ToIntFunction; - -import static com.google.common.collect.Maps.uniqueIndex; -import static io.trino.spi.StandardErrorCode.NO_NODES_AVAILABLE; -import static io.trino.spi.connector.ConnectorBucketNodeMap.createBucketNodeMap; -import static java.util.Objects.requireNonNull; - -public class RaptorNodePartitioningProvider - implements ConnectorNodePartitioningProvider -{ - private final NodeSupplier nodeSupplier; - - @Inject - public RaptorNodePartitioningProvider(NodeSupplier nodeSupplier) - { - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - } - - @Override - public Optional getBucketNodeMapping(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorPartitioningHandle partitioning) - { - if (partitioning instanceof RaptorUnbucketedUpdateHandle) { - return Optional.empty(); - } - - RaptorPartitioningHandle handle = (RaptorPartitioningHandle) partitioning; - - Map nodesById = uniqueIndex(nodeSupplier.getWorkerNodes(), Node::getNodeIdentifier); - - ImmutableList.Builder bucketToNode = ImmutableList.builder(); - for (String nodeIdentifier : handle.getBucketToNode()) { - Node node = nodesById.get(nodeIdentifier); - if (node == null) { - throw new TrinoException(NO_NODES_AVAILABLE, "Node for bucket is offline: " + nodeIdentifier); - } - bucketToNode.add(node); - } - return Optional.of(createBucketNodeMap(bucketToNode.build())); - } - - @Override - public ToIntFunction getSplitBucketFunction(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorPartitioningHandle partitioning) - { - return value -> ((RaptorSplit) value).getBucketNumber().getAsInt(); - } - - @Override - public BucketFunction getBucketFunction(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorPartitioningHandle partitioning, List partitionChannelTypes, int bucketCount) - { - if (partitioning instanceof RaptorUnbucketedUpdateHandle) { - return new RaptorUnbucketedUpdateFunction(bucketCount); - } - if (partitioning instanceof RaptorBucketedUpdateHandle) { - return new RaptorBucketedUpdateFunction(); - } - return new RaptorBucketFunction(bucketCount, partitionChannelTypes); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorOutputTableHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorOutputTableHandle.java deleted file mode 100644 index e512eb06fcee..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorOutputTableHandle.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static io.trino.plugin.raptor.legacy.util.MetadataUtil.checkSchemaName; -import static io.trino.plugin.raptor.legacy.util.MetadataUtil.checkTableName; -import static java.util.Objects.requireNonNull; - -public class RaptorOutputTableHandle - implements ConnectorOutputTableHandle -{ - private final long transactionId; - private final String schemaName; - private final String tableName; - private final List columnHandles; - private final List columnTypes; - private final List sortColumnHandles; - private final List sortOrders; - private final Optional temporalColumnHandle; - private final OptionalLong distributionId; - private final OptionalInt bucketCount; - private final List bucketColumnHandles; - private final boolean organized; - - @JsonCreator - public RaptorOutputTableHandle( - @JsonProperty("transactionId") long transactionId, - @JsonProperty("schemaName") String schemaName, - @JsonProperty("tableName") String tableName, - @JsonProperty("columnHandles") List columnHandles, - @JsonProperty("columnTypes") List columnTypes, - @JsonProperty("sortColumnHandles") List sortColumnHandles, - @JsonProperty("sortOrders") List sortOrders, - @JsonProperty("temporalColumnHandle") Optional temporalColumnHandle, - @JsonProperty("distributionId") OptionalLong distributionId, - @JsonProperty("bucketCount") OptionalInt bucketCount, - @JsonProperty("organized") boolean organized, - @JsonProperty("bucketColumnHandles") List bucketColumnHandles) - { - this.transactionId = transactionId; - this.schemaName = checkSchemaName(schemaName); - this.tableName = checkTableName(tableName); - this.columnHandles = ImmutableList.copyOf(requireNonNull(columnHandles, "columnHandles is null")); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.sortOrders = requireNonNull(sortOrders, "sortOrders is null"); - this.sortColumnHandles = requireNonNull(sortColumnHandles, "sortColumnHandles is null"); - this.temporalColumnHandle = requireNonNull(temporalColumnHandle, "temporalColumnHandle is null"); - this.distributionId = requireNonNull(distributionId, "distributionId is null"); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.bucketColumnHandles = ImmutableList.copyOf(requireNonNull(bucketColumnHandles, "bucketColumnHandles is null")); - this.organized = organized; - } - - @JsonProperty - public long getTransactionId() - { - return transactionId; - } - - @JsonProperty - public String getSchemaName() - { - return schemaName; - } - - @JsonProperty - public String getTableName() - { - return tableName; - } - - @JsonProperty - public List getColumnHandles() - { - return columnHandles; - } - - @JsonProperty - public List getColumnTypes() - { - return columnTypes; - } - - @JsonProperty - public List getSortColumnHandles() - { - return sortColumnHandles; - } - - @JsonProperty - public List getSortOrders() - { - return sortOrders; - } - - @JsonProperty - public Optional getTemporalColumnHandle() - { - return temporalColumnHandle; - } - - @JsonProperty - public OptionalLong getDistributionId() - { - return distributionId; - } - - @JsonProperty - public OptionalInt getBucketCount() - { - return bucketCount; - } - - @JsonProperty - public List getBucketColumnHandles() - { - return bucketColumnHandles; - } - - @JsonProperty - public boolean isOrganized() - { - return organized; - } - - @Override - public String toString() - { - return "raptor:" + schemaName + "." + tableName; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSink.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSink.java deleted file mode 100644 index 067002f98536..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSink.java +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.organization.TemporalFunction; -import io.trino.plugin.raptor.legacy.util.PageBuffer; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.PageSorter; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.type.Type; -import it.unimi.dsi.fastutil.longs.Long2ObjectMap; -import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.concurrent.CompletableFuture; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class RaptorPageSink - implements ConnectorPageSink -{ - private static final JsonCodec SHARD_INFO_CODEC = jsonCodec(ShardInfo.class); - - private final long transactionId; - private final StorageManager storageManager; - private final PageSorter pageSorter; - private final List columnIds; - private final List columnTypes; - private final List sortFields; - private final List sortOrders; - private final OptionalInt bucketCount; - private final int[] bucketFields; - private final long maxBufferBytes; - private final OptionalInt temporalColumnIndex; - private final Optional temporalColumnType; - - private final PageWriter pageWriter; - - public RaptorPageSink( - PageSorter pageSorter, - StorageManager storageManager, - long transactionId, - List columnIds, - List columnTypes, - List sortColumnIds, - List sortOrders, - OptionalInt bucketCount, - List bucketColumnIds, - Optional temporalColumnHandle, - DataSize maxBufferSize) - { - this.transactionId = transactionId; - this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); - this.columnIds = ImmutableList.copyOf(requireNonNull(columnIds, "columnIds is null")); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - this.maxBufferBytes = maxBufferSize.toBytes(); - - this.sortFields = sortColumnIds.stream().map(columnIds::indexOf).collect(toImmutableList()); - this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); - - this.bucketCount = bucketCount; - this.bucketFields = bucketColumnIds.stream().mapToInt(columnIds::indexOf).toArray(); - - if (temporalColumnHandle.isPresent() && columnIds.contains(temporalColumnHandle.get().getColumnId())) { - temporalColumnIndex = OptionalInt.of(columnIds.indexOf(temporalColumnHandle.get().getColumnId())); - temporalColumnType = Optional.of(columnTypes.get(temporalColumnIndex.getAsInt())); - checkArgument(temporalColumnType.get() == DATE || temporalColumnType.get().equals(TIMESTAMP_MILLIS), - "temporalColumnType can only be DATE or TIMESTAMP"); - } - else { - temporalColumnIndex = OptionalInt.empty(); - temporalColumnType = Optional.empty(); - } - - this.pageWriter = (bucketCount.isPresent() || temporalColumnIndex.isPresent()) ? new PartitionedPageWriter() : new SimplePageWriter(); - } - - @Override - public CompletableFuture appendPage(Page page) - { - if (page.getPositionCount() == 0) { - return NOT_BLOCKED; - } - - pageWriter.appendPage(page); - return NOT_BLOCKED; - } - - @Override - public CompletableFuture> finish() - { - List>> futureSlices = pageWriter.getPageBuffers().stream().map(pageBuffer -> { - pageBuffer.flush(); - CompletableFuture> futureShards = pageBuffer.getStoragePageSink().commit(); - return futureShards.thenApply(shards -> shards.stream() - .map(shard -> Slices.wrappedBuffer(SHARD_INFO_CODEC.toJsonBytes(shard))) - .collect(toList())); - }).collect(toList()); - - return allAsList(futureSlices).thenApply(lists -> lists.stream() - .flatMap(Collection::stream) - .collect(toList())); - } - - @Override - public void abort() - { - RuntimeException error = new RuntimeException("Exception during rollback"); - for (PageBuffer pageBuffer : pageWriter.getPageBuffers()) { - try { - pageBuffer.getStoragePageSink().rollback(); - } - catch (Throwable t) { - // Self-suppression not permitted - if (error != t) { - error.addSuppressed(t); - } - } - } - if (error.getSuppressed().length > 0) { - throw error; - } - } - - private PageBuffer createPageBuffer(OptionalInt bucketNumber) - { - return new PageBuffer( - maxBufferBytes, - storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, true), - columnTypes, - sortFields, - sortOrders, - pageSorter); - } - - private interface PageWriter - { - void appendPage(Page page); - - List getPageBuffers(); - } - - private class SimplePageWriter - implements PageWriter - { - private final PageBuffer pageBuffer = createPageBuffer(OptionalInt.empty()); - - @Override - public void appendPage(Page page) - { - pageBuffer.add(page); - } - - @Override - public List getPageBuffers() - { - return ImmutableList.of(pageBuffer); - } - } - - private class PartitionedPageWriter - implements PageWriter - { - private final Optional bucketFunction; - private final Long2ObjectMap pageStores = new Long2ObjectOpenHashMap<>(); - - public PartitionedPageWriter() - { - checkArgument(temporalColumnIndex.isPresent() == temporalColumnType.isPresent(), - "temporalColumnIndex and temporalColumnType must be both present or absent"); - - List bucketTypes = Arrays.stream(bucketFields) - .mapToObj(columnTypes::get) - .collect(toList()); - - this.bucketFunction = bucketCount.isPresent() ? Optional.of(new RaptorBucketFunction(bucketCount.getAsInt(), bucketTypes)) : Optional.empty(); - } - - @Override - public void appendPage(Page page) - { - Block temporalBlock = temporalColumnIndex.isPresent() ? page.getBlock(temporalColumnIndex.getAsInt()) : null; - - Page bucketArgs = bucketFunction.isPresent() ? getBucketArgsPage(page) : null; - - for (int position = 0; position < page.getPositionCount(); position++) { - int bucket = bucketFunction.isPresent() ? bucketFunction.get().getBucket(bucketArgs, position) : 0; - int day = temporalColumnType.isPresent() ? TemporalFunction.getDay(temporalColumnType.get(), temporalBlock, position) : 0; - - long partition = (((long) bucket) << 32) | (day & 0xFFFF_FFFFL); - PageStore store = pageStores.get(partition); - if (store == null) { - OptionalInt bucketNumber = bucketFunction.isPresent() ? OptionalInt.of(bucket) : OptionalInt.empty(); - PageBuffer buffer = createPageBuffer(bucketNumber); - store = new PageStore(buffer, columnTypes); - pageStores.put(partition, store); - } - - store.appendPosition(page, position); - } - - flushIfNecessary(); - } - - private Page getBucketArgsPage(Page page) - { - Block[] blocks = new Block[bucketFields.length]; - for (int i = 0; i < bucketFields.length; i++) { - blocks[i] = page.getBlock(bucketFields[i]); - } - return new Page(page.getPositionCount(), blocks); - } - - @Override - public List getPageBuffers() - { - ImmutableList.Builder list = ImmutableList.builder(); - for (PageStore store : pageStores.values()) { - store.flushToPageBuffer(); - store.getPageBuffer().flush(); - list.add(store.getPageBuffer()); - } - return list.build(); - } - - private void flushIfNecessary() - { - long totalBytes = 0; - long maxBytes = 0; - PageBuffer maxBuffer = null; - - for (PageStore store : pageStores.values()) { - long bytes = store.getUsedMemoryBytes(); - totalBytes += bytes; - - if ((maxBuffer == null) || (bytes > maxBytes)) { - maxBuffer = store.getPageBuffer(); - maxBytes = bytes; - } - } - - if ((totalBytes > maxBufferBytes) && (maxBuffer != null)) { - maxBuffer.flush(); - } - } - } - - private static class PageStore - { - private final PageBuffer pageBuffer; - private final PageBuilder pageBuilder; - - public PageStore(PageBuffer pageBuffer, List columnTypes) - { - this.pageBuffer = requireNonNull(pageBuffer, "pageBuffer is null"); - this.pageBuilder = new PageBuilder(columnTypes); - } - - public long getUsedMemoryBytes() - { - return pageBuilder.getSizeInBytes() + pageBuffer.getUsedMemoryBytes(); - } - - public PageBuffer getPageBuffer() - { - return pageBuffer; - } - - public void appendPosition(Page page, int position) - { - pageBuilder.declarePosition(); - for (int channel = 0; channel < page.getChannelCount(); channel++) { - Block block = page.getBlock(channel); - BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(channel); - pageBuilder.getType(channel).appendTo(block, position, blockBuilder); - } - - if (pageBuilder.isFull()) { - flushToPageBuffer(); - } - } - - public void flushToPageBuffer() - { - if (!pageBuilder.isEmpty()) { - pageBuffer.add(pageBuilder.build()); - pageBuilder.reset(); - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSinkProvider.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSinkProvider.java deleted file mode 100644 index 7a32548b4fd9..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSinkProvider.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.inject.Inject; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.PageSorter; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorMergeSink; -import io.trino.spi.connector.ConnectorMergeTableHandle; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.ConnectorPageSinkId; -import io.trino.spi.connector.ConnectorPageSinkProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTransactionHandle; - -import java.util.List; - -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class RaptorPageSinkProvider - implements ConnectorPageSinkProvider -{ - private final StorageManager storageManager; - private final PageSorter pageSorter; - private final DataSize maxBufferSize; - - @Inject - public RaptorPageSinkProvider(StorageManager storageManager, PageSorter pageSorter, StorageManagerConfig config) - { - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); - this.maxBufferSize = config.getMaxBufferSize(); - } - - @Override - public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle tableHandle, ConnectorPageSinkId pageSinkId) - { - RaptorOutputTableHandle handle = (RaptorOutputTableHandle) tableHandle; - return new RaptorPageSink( - pageSorter, - storageManager, - handle.getTransactionId(), - toColumnIds(handle.getColumnHandles()), - handle.getColumnTypes(), - toColumnIds(handle.getSortColumnHandles()), - handle.getSortOrders(), - handle.getBucketCount(), - toColumnIds(handle.getBucketColumnHandles()), - handle.getTemporalColumnHandle(), - maxBufferSize); - } - - @Override - public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle tableHandle, ConnectorPageSinkId pageSinkId) - { - RaptorInsertTableHandle handle = (RaptorInsertTableHandle) tableHandle; - return new RaptorPageSink( - pageSorter, - storageManager, - handle.getTransactionId(), - toColumnIds(handle.getColumnHandles()), - handle.getColumnTypes(), - toColumnIds(handle.getSortColumnHandles()), - handle.getSortOrders(), - handle.getBucketCount(), - toColumnIds(handle.getBucketColumnHandles()), - handle.getTemporalColumnHandle(), - maxBufferSize); - } - - @Override - public ConnectorMergeSink createMergeSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorMergeTableHandle mergeHandle, ConnectorPageSinkId pageSinkId) - { - RaptorMergeTableHandle merge = (RaptorMergeTableHandle) mergeHandle; - ConnectorPageSink pageSink = createPageSink(transactionHandle, session, merge.getInsertTableHandle(), pageSinkId); - long transactionId = merge.getInsertTableHandle().getTransactionId(); - int columnCount = merge.getInsertTableHandle().getColumnHandles().size(); - return new RaptorMergeSink(pageSink, storageManager, transactionId, columnCount); - } - - private static List toColumnIds(List columnHandles) - { - return columnHandles.stream().map(RaptorColumnHandle::getColumnId).collect(toList()); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSourceProvider.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSourceProvider.java deleted file mode 100644 index e266d8562e73..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSourceProvider.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.inject.Inject; -import io.trino.orc.OrcReaderOptions; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.util.ConcatPageSource; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.ConnectorPageSourceProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; - -import java.util.Iterator; -import java.util.List; -import java.util.OptionalInt; -import java.util.UUID; - -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderMaxMergeDistance; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderMaxReadSize; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderStreamBufferSize; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderTinyStripeThreshold; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.isReaderLazyReadSmallRanges; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class RaptorPageSourceProvider - implements ConnectorPageSourceProvider -{ - private final StorageManager storageManager; - - @Inject - public RaptorPageSourceProvider(StorageManager storageManager) - { - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - } - - @Override - public ConnectorPageSource createPageSource( - ConnectorTransactionHandle transaction, - ConnectorSession session, - ConnectorSplit split, - ConnectorTableHandle table, - List columns, - DynamicFilter dynamicFilter) - { - RaptorSplit raptorSplit = (RaptorSplit) split; - RaptorTableHandle raptorTable = (RaptorTableHandle) table; - - OptionalInt bucketNumber = raptorSplit.getBucketNumber(); - TupleDomain predicate = raptorTable.getConstraint(); - OrcReaderOptions options = new OrcReaderOptions() - .withMaxMergeDistance(getReaderMaxMergeDistance(session)) - .withMaxBufferSize(getReaderMaxReadSize(session)) - .withStreamBufferSize(getReaderStreamBufferSize(session)) - .withTinyStripeThreshold(getReaderTinyStripeThreshold(session)) - .withLazyReadSmallRanges(isReaderLazyReadSmallRanges(session)); - - if (raptorSplit.getShardUuids().size() == 1) { - UUID shardUuid = raptorSplit.getShardUuids().iterator().next(); - return createPageSource(shardUuid, bucketNumber, columns, predicate, options); - } - - Iterator iterator = raptorSplit.getShardUuids().stream() - .map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, options)) - .iterator(); - - return new ConcatPageSource(iterator); - } - - private ConnectorPageSource createPageSource( - UUID shardUuid, - OptionalInt bucketNumber, - List columns, - TupleDomain predicate, - OrcReaderOptions orcReaderOptions) - { - List columnHandles = columns.stream().map(RaptorColumnHandle.class::cast).collect(toList()); - List columnIds = columnHandles.stream().map(RaptorColumnHandle::getColumnId).collect(toList()); - List columnTypes = columnHandles.stream().map(RaptorColumnHandle::getColumnType).collect(toList()); - - return storageManager.getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, predicate, orcReaderOptions); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPartitioningHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPartitioningHandle.java deleted file mode 100644 index ca6055e1bcd2..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPartitioningHandle.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ConnectorPartitioningHandle; - -import java.util.List; -import java.util.Objects; - -import static java.util.Objects.requireNonNull; - -public class RaptorPartitioningHandle - implements ConnectorPartitioningHandle -{ - private final long distributionId; - private final List bucketToNode; - - @JsonCreator - public RaptorPartitioningHandle( - @JsonProperty("distributionId") long distributionId, - @JsonProperty("bucketToNode") List bucketToNode) - { - this.distributionId = distributionId; - this.bucketToNode = ImmutableList.copyOf(requireNonNull(bucketToNode, "bucketToNode is null")); - } - - @JsonProperty - public long getDistributionId() - { - return distributionId; - } - - @JsonProperty - public List getBucketToNode() - { - return bucketToNode; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if ((o == null) || (getClass() != o.getClass())) { - return false; - } - RaptorPartitioningHandle that = (RaptorPartitioningHandle) o; - return (distributionId == that.distributionId) && - Objects.equals(bucketToNode, that.bucketToNode); - } - - @Override - public int hashCode() - { - return Objects.hash(distributionId, bucketToNode); - } - - @Override - public String toString() - { - return String.valueOf(distributionId); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPlugin.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPlugin.java deleted file mode 100644 index d2159a946dcc..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPlugin.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.inject.Module; -import io.trino.plugin.raptor.legacy.metadata.DatabaseMetadataModule; -import io.trino.spi.Plugin; -import io.trino.spi.connector.ConnectorFactory; - -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.util.Objects.requireNonNull; - -public class RaptorPlugin - implements Plugin -{ - private final String name; - private final Module metadataModule; - private final Map backupProviders; - - public RaptorPlugin() - { - this("raptor_legacy", new DatabaseMetadataModule(), ImmutableMap.of()); - } - - public RaptorPlugin(String name, Module metadataModule, Map backupProviders) - { - checkArgument(!isNullOrEmpty(name), "name is null or empty"); - this.name = name; - this.metadataModule = requireNonNull(metadataModule, "metadataModule is null"); - this.backupProviders = ImmutableMap.copyOf(requireNonNull(backupProviders, "backupProviders is null")); - } - - @Override - public Iterable getConnectorFactories() - { - return ImmutableList.of(new RaptorConnectorFactory(name, metadataModule, backupProviders)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSessionProperties.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSessionProperties.java deleted file mode 100644 index f9f47e1edf25..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSessionProperties.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.session.PropertyMetadata; - -import java.util.List; -import java.util.Optional; - -import static io.trino.plugin.base.session.PropertyMetadataUtil.dataSizeProperty; -import static io.trino.spi.session.PropertyMetadata.booleanProperty; -import static io.trino.spi.session.PropertyMetadata.integerProperty; -import static io.trino.spi.session.PropertyMetadata.stringProperty; - -public class RaptorSessionProperties -{ - private static final String EXTERNAL_BATCH_ID = "external_batch_id"; - - private static final String READER_MAX_MERGE_DISTANCE = "reader_max_merge_distance"; - private static final String READER_MAX_READ_SIZE = "reader_max_read_size"; - private static final String READER_STREAM_BUFFER_SIZE = "reader_stream_buffer_size"; - private static final String READER_TINY_STRIPE_THRESHOLD = "reader_tiny_stripe_threshold"; - private static final String READER_LAZY_READ_SMALL_RANGES = "reader_lazy_read_small_ranges"; - private static final String ONE_SPLIT_PER_BUCKET_THRESHOLD = "one_split_per_bucket_threshold"; - - private final List> sessionProperties; - - @Inject - public RaptorSessionProperties(StorageManagerConfig config) - { - sessionProperties = ImmutableList.of( - stringProperty( - EXTERNAL_BATCH_ID, - "Two-phase commit batch ID", - null, - true), - dataSizeProperty( - READER_MAX_MERGE_DISTANCE, - "Reader: Maximum size of gap between two reads to merge into a single read", - config.getOrcMaxMergeDistance(), - false), - dataSizeProperty( - READER_MAX_READ_SIZE, - "Reader: Maximum size of a single read", - config.getOrcMaxReadSize(), - false), - dataSizeProperty( - READER_STREAM_BUFFER_SIZE, - "Reader: Size of buffer for streaming reads", - config.getOrcStreamBufferSize(), - false), - dataSizeProperty( - READER_TINY_STRIPE_THRESHOLD, - "Reader: Threshold below which an ORC stripe or file will read in its entirety", - config.getOrcTinyStripeThreshold(), - false), - booleanProperty( - READER_LAZY_READ_SMALL_RANGES, - "Experimental: Reader: Read small file segments lazily", - config.isOrcLazyReadSmallRanges(), - false), - integerProperty( - ONE_SPLIT_PER_BUCKET_THRESHOLD, - "Experimental: Maximum bucket count at which to produce multiple splits per bucket", - config.getOneSplitPerBucketThreshold(), - false)); - } - - public List> getSessionProperties() - { - return sessionProperties; - } - - public static Optional getExternalBatchId(ConnectorSession session) - { - return Optional.ofNullable(session.getProperty(EXTERNAL_BATCH_ID, String.class)); - } - - public static DataSize getReaderMaxMergeDistance(ConnectorSession session) - { - return session.getProperty(READER_MAX_MERGE_DISTANCE, DataSize.class); - } - - public static DataSize getReaderMaxReadSize(ConnectorSession session) - { - return session.getProperty(READER_MAX_READ_SIZE, DataSize.class); - } - - public static DataSize getReaderStreamBufferSize(ConnectorSession session) - { - return session.getProperty(READER_STREAM_BUFFER_SIZE, DataSize.class); - } - - public static DataSize getReaderTinyStripeThreshold(ConnectorSession session) - { - return session.getProperty(READER_TINY_STRIPE_THRESHOLD, DataSize.class); - } - - public static boolean isReaderLazyReadSmallRanges(ConnectorSession session) - { - return session.getProperty(READER_LAZY_READ_SMALL_RANGES, Boolean.class); - } - - public static int getOneSplitPerBucketThreshold(ConnectorSession session) - { - return session.getProperty(ONE_SPLIT_PER_BUCKET_THRESHOLD, Integer.class); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplit.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplit.java deleted file mode 100644 index f2198b73e973..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplit.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.trino.spi.HostAddress; -import io.trino.spi.connector.ConnectorSplit; - -import java.util.List; -import java.util.Map; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.airlift.slice.SizeOf.estimatedSizeOf; -import static io.airlift.slice.SizeOf.instanceSize; -import static io.airlift.slice.SizeOf.sizeOf; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.joining; - -public class RaptorSplit - implements ConnectorSplit -{ - private static final int INSTANCE_SIZE = instanceSize(RaptorSplit.class); - private static final int UUID_INSTANCE_SIZE = instanceSize(UUID.class); - - private final Set shardUuids; - private final OptionalInt bucketNumber; - private final List addresses; - - @JsonCreator - public RaptorSplit( - @JsonProperty("shardUuids") Set shardUuids, - @JsonProperty("bucketNumber") OptionalInt bucketNumber) - { - this(shardUuids, bucketNumber, ImmutableList.of()); - } - - public RaptorSplit(UUID shardUuid, List addresses) - { - this(ImmutableSet.of(shardUuid), OptionalInt.empty(), addresses); - } - - public RaptorSplit(Set shardUuids, int bucketNumber, HostAddress address) - { - this(shardUuids, OptionalInt.of(bucketNumber), ImmutableList.of(address)); - } - - private RaptorSplit(Set shardUuids, OptionalInt bucketNumber, List addresses) - { - this.shardUuids = ImmutableSet.copyOf(requireNonNull(shardUuids, "shardUuids is null")); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.addresses = ImmutableList.copyOf(requireNonNull(addresses, "addresses is null")); - } - - @Override - public boolean isRemotelyAccessible() - { - return false; - } - - @Override - public List getAddresses() - { - return addresses; - } - - @JsonProperty - public Set getShardUuids() - { - return shardUuids; - } - - @JsonProperty - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - @Override - public Map getSplitInfo() - { - return ImmutableMap.of( - "addresses", addresses.stream().map(HostAddress::toString).collect(joining(",")), - "bucketNumber", bucketNumber.isPresent() ? String.valueOf(bucketNumber.getAsInt()) : "", - "shardUuids", shardUuids.stream().map(UUID::toString).collect(joining(","))); - } - - @Override - public long getRetainedSizeInBytes() - { - return INSTANCE_SIZE - + estimatedSizeOf(shardUuids, value -> UUID_INSTANCE_SIZE) - + sizeOf(bucketNumber) - + estimatedSizeOf(addresses, HostAddress::getRetainedSizeInBytes); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuids", shardUuids) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .add("hosts", addresses) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplitManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplitManager.java deleted file mode 100644 index e480bdd2b67d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplitManager.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.backup.BackupService; -import io.trino.plugin.raptor.legacy.metadata.BucketShards; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardNodes; -import io.trino.plugin.raptor.legacy.util.SynchronizedResultIterator; -import io.trino.spi.HostAddress; -import io.trino.spi.Node; -import io.trino.spi.TrinoException; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorSplitSource; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.predicate.TupleDomain; -import jakarta.annotation.PreDestroy; -import org.jdbi.v3.core.result.ResultIterator; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.function.Supplier; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.collect.Maps.uniqueIndex; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_NO_HOST_FOR_SHARD; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getOneSplitPerBucketThreshold; -import static io.trino.spi.StandardErrorCode.NO_NODES_AVAILABLE; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.supplyAsync; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.stream.Collectors.toSet; - -public class RaptorSplitManager - implements ConnectorSplitManager -{ - private final NodeSupplier nodeSupplier; - private final ShardManager shardManager; - private final boolean backupAvailable; - private final ExecutorService executor; - - @Inject - public RaptorSplitManager(CatalogName catalogName, NodeSupplier nodeSupplier, ShardManager shardManager, BackupService backupService) - { - this(catalogName, nodeSupplier, shardManager, backupService.isBackupAvailable()); - } - - public RaptorSplitManager(CatalogName catalogName, NodeSupplier nodeSupplier, ShardManager shardManager, boolean backupAvailable) - { - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.backupAvailable = backupAvailable; - this.executor = newCachedThreadPool(daemonThreadsNamed("raptor-split-" + catalogName + "-%s")); - } - - @PreDestroy - public void destroy() - { - executor.shutdownNow(); - } - - @Override - public ConnectorSplitSource getSplits( - ConnectorTransactionHandle transaction, - ConnectorSession session, - ConnectorTableHandle handle, - DynamicFilter dynamicFilter, - Constraint constraint) - { - RaptorTableHandle table = (RaptorTableHandle) handle; - long tableId = table.getTableId(); - boolean bucketed = table.getBucketCount().isPresent(); - boolean merged = bucketed && (table.getBucketCount().getAsInt() >= getOneSplitPerBucketThreshold(session)); - Optional> bucketToNode = table.getBucketAssignments(); - verify(bucketed == bucketToNode.isPresent(), "mismatched bucketCount and bucketToNode presence"); - return new RaptorSplitSource(tableId, merged, table.getConstraint(), bucketToNode); - } - - private static List getAddressesForNodes(Map nodeMap, Iterable nodeIdentifiers) - { - ImmutableList.Builder nodes = ImmutableList.builder(); - for (String id : nodeIdentifiers) { - Node node = nodeMap.get(id); - if (node != null) { - nodes.add(node.getHostAndPort()); - } - } - return nodes.build(); - } - - private static T selectRandom(Iterable elements) - { - List list = ImmutableList.copyOf(elements); - return list.get(ThreadLocalRandom.current().nextInt(list.size())); - } - - private class RaptorSplitSource - implements ConnectorSplitSource - { - private final Map nodesById = uniqueIndex(nodeSupplier.getWorkerNodes(), Node::getNodeIdentifier); - private final long tableId; - private final Optional> bucketToNode; - private final ResultIterator iterator; - - @GuardedBy("this") - private CompletableFuture future; - - public RaptorSplitSource( - long tableId, - boolean merged, - TupleDomain effectivePredicate, - Optional> bucketToNode) - { - this.tableId = tableId; - this.bucketToNode = requireNonNull(bucketToNode, "bucketToNode is null"); - - ResultIterator iterator; - if (bucketToNode.isPresent()) { - iterator = shardManager.getShardNodesBucketed(tableId, merged, bucketToNode.get(), effectivePredicate); - } - else { - iterator = shardManager.getShardNodes(tableId, effectivePredicate); - } - this.iterator = new SynchronizedResultIterator<>(iterator); - } - - @Override - public synchronized CompletableFuture getNextBatch(int maxSize) - { - checkState((future == null) || future.isDone(), "previous batch not completed"); - future = supplyAsync(batchSupplier(maxSize), executor); - return future; - } - - @Override - public synchronized void close() - { - if (future != null) { - future.cancel(true); - future = null; - } - executor.execute(iterator::close); - } - - @Override - public boolean isFinished() - { - return !iterator.hasNext(); - } - - private Supplier batchSupplier(int maxSize) - { - return () -> { - ImmutableList.Builder list = ImmutableList.builder(); - for (int i = 0; i < maxSize; i++) { - if (Thread.currentThread().isInterrupted()) { - throw new RuntimeException("Split batch fetch was interrupted"); - } - if (!iterator.hasNext()) { - break; - } - list.add(createSplit(iterator.next())); - } - return new ConnectorSplitBatch(list.build(), isFinished()); - }; - } - - private ConnectorSplit createSplit(BucketShards bucketShards) - { - if (bucketShards.getBucketNumber().isPresent()) { - return createBucketSplit(bucketShards.getBucketNumber().getAsInt(), bucketShards.getShards()); - } - - verify(bucketShards.getShards().size() == 1, "wrong shard count for non-bucketed table"); - ShardNodes shard = getOnlyElement(bucketShards.getShards()); - UUID shardId = shard.getShardUuid(); - Set nodeIds = shard.getNodeIdentifiers(); - - List addresses = getAddressesForNodes(nodesById, nodeIds); - if (addresses.isEmpty()) { - if (!backupAvailable) { - throw new TrinoException(RAPTOR_NO_HOST_FOR_SHARD, format("No host for shard %s found: %s", shardId, nodeIds)); - } - - // Pick a random node and optimistically assign the shard to it. - // That node will restore the shard from the backup location. - Set availableNodes = nodeSupplier.getWorkerNodes(); - if (availableNodes.isEmpty()) { - throw new TrinoException(NO_NODES_AVAILABLE, "No nodes available to run query"); - } - Node node = selectRandom(availableNodes); - shardManager.replaceShardAssignment(tableId, shardId, node.getNodeIdentifier(), true); - addresses = ImmutableList.of(node.getHostAndPort()); - } - - return new RaptorSplit(shardId, addresses); - } - - private ConnectorSplit createBucketSplit(int bucketNumber, Set shards) - { - // Bucket splits contain all the shards for the bucket - // and run on the node assigned to the bucket. - - String nodeId = bucketToNode.get().get(bucketNumber); - Node node = nodesById.get(nodeId); - if (node == null) { - throw new TrinoException(NO_NODES_AVAILABLE, "Node for bucket is offline: " + nodeId); - } - - Set shardUuids = shards.stream() - .map(ShardNodes::getShardUuid) - .collect(toSet()); - HostAddress address = node.getHostAndPort(); - - return new RaptorSplit(shardUuids, bucketNumber, address); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableHandle.java deleted file mode 100644 index 4bb583bc3771..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableHandle.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.predicate.TupleDomain; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.raptor.legacy.util.MetadataUtil.checkSchemaName; -import static io.trino.plugin.raptor.legacy.util.MetadataUtil.checkTableName; -import static java.util.Objects.requireNonNull; - -public final class RaptorTableHandle - implements ConnectorTableHandle -{ - private final String schemaName; - private final String tableName; - private final long tableId; - private final Optional distributionId; - private final Optional distributionName; - private final OptionalInt bucketCount; - private final boolean organized; - private final TupleDomain constraint; - private final Optional> bucketAssignments; - - @JsonCreator - public RaptorTableHandle( - @JsonProperty("schemaName") String schemaName, - @JsonProperty("tableName") String tableName, - @JsonProperty("tableId") long tableId, - @JsonProperty("distributionId") Optional distributionId, - @JsonProperty("distributionName") Optional distributionName, - @JsonProperty("bucketCount") OptionalInt bucketCount, - @JsonProperty("organized") boolean organized, - @JsonProperty("constraint") TupleDomain constraint, - // this field will not be in the JSON, but keep it here to avoid duplicating the constructor - @JsonProperty("bucketAssignments") Optional> bucketAssignments) - { - this.schemaName = checkSchemaName(schemaName); - this.tableName = checkTableName(tableName); - - checkArgument(tableId > 0, "tableId must be greater than zero"); - this.tableId = tableId; - - this.distributionName = requireNonNull(distributionName, "distributionName is null"); - this.distributionId = requireNonNull(distributionId, "distributionId is null"); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.organized = organized; - - this.constraint = requireNonNull(constraint, "constraint is null"); - this.bucketAssignments = bucketAssignments.map(ImmutableList::copyOf); - } - - public boolean isBucketed() - { - return this.distributionId.isPresent(); - } - - @JsonProperty - public String getSchemaName() - { - return schemaName; - } - - @JsonProperty - public String getTableName() - { - return tableName; - } - - @JsonProperty - public long getTableId() - { - return tableId; - } - - @JsonProperty - public Optional getDistributionId() - { - return distributionId; - } - - @JsonProperty - public Optional getDistributionName() - { - return distributionName; - } - - @JsonProperty - public OptionalInt getBucketCount() - { - return bucketCount; - } - - @JsonProperty - public boolean isOrganized() - { - return organized; - } - - @JsonProperty - public TupleDomain getConstraint() - { - return constraint; - } - - @JsonIgnore - public Optional> getBucketAssignments() - { - return bucketAssignments; - } - - @Override - public String toString() - { - return schemaName + ":" + tableName + ":" + tableId; - } - - @Override - public int hashCode() - { - return Objects.hash(schemaName, tableName, tableId, distributionId, distributionName, bucketCount, organized, constraint, bucketAssignments); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - RaptorTableHandle other = (RaptorTableHandle) obj; - return Objects.equals(this.schemaName, other.schemaName) && - Objects.equals(this.tableName, other.tableName) && - this.tableId == other.tableId && - Objects.equals(this.distributionId, other.distributionId) && - Objects.equals(this.distributionName, other.distributionName) && - Objects.equals(this.bucketCount, other.bucketCount) && - this.organized == other.organized && - Objects.equals(this.constraint, other.constraint) && - Objects.equals(this.bucketAssignments, other.bucketAssignments); - } - - @JsonIgnore - public Optional getPartitioningHandle() - { - return distributionId.map(id -> new RaptorPartitioningHandle(id, bucketAssignments.get())); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableProperties.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableProperties.java deleted file mode 100644 index 293f89a602f2..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableProperties.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.trino.spi.session.PropertyMetadata; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeSignatureParameter; - -import java.util.List; -import java.util.Map; -import java.util.OptionalInt; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.spi.session.PropertyMetadata.booleanProperty; -import static io.trino.spi.session.PropertyMetadata.integerProperty; -import static io.trino.spi.type.StandardTypes.ARRAY; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static java.util.Locale.ENGLISH; - -public class RaptorTableProperties -{ - public static final String ORDERING_PROPERTY = "ordering"; - public static final String TEMPORAL_COLUMN_PROPERTY = "temporal_column"; - public static final String BUCKET_COUNT_PROPERTY = "bucket_count"; - public static final String BUCKETED_ON_PROPERTY = "bucketed_on"; - public static final String DISTRIBUTION_NAME_PROPERTY = "distribution_name"; - public static final String ORGANIZED_PROPERTY = "organized"; - - private final List> tableProperties; - - @Inject - public RaptorTableProperties(TypeManager typeManager) - { - tableProperties = ImmutableList.>builder() - .add(stringListSessionProperty( - typeManager, - ORDERING_PROPERTY, - "Sort order for each shard of the table")) - .add(lowerCaseStringSessionProperty( - TEMPORAL_COLUMN_PROPERTY, - "Temporal column of the table")) - .add(integerProperty( - BUCKET_COUNT_PROPERTY, - "Number of buckets into which to divide the table", - null, - false)) - .add(stringListSessionProperty( - typeManager, - BUCKETED_ON_PROPERTY, - "Table columns on which to bucket the table")) - .add(lowerCaseStringSessionProperty( - DISTRIBUTION_NAME_PROPERTY, - "Shared distribution name for colocated tables")) - .add(booleanProperty( - ORGANIZED_PROPERTY, - "Keep the table organized using the sort order", - null, - false)) - .build(); - } - - public List> getTableProperties() - { - return tableProperties; - } - - public static List getSortColumns(Map tableProperties) - { - return stringList(tableProperties.get(ORDERING_PROPERTY)); - } - - public static String getTemporalColumn(Map tableProperties) - { - return (String) tableProperties.get(TEMPORAL_COLUMN_PROPERTY); - } - - public static OptionalInt getBucketCount(Map tableProperties) - { - Integer value = (Integer) tableProperties.get(BUCKET_COUNT_PROPERTY); - return (value != null) ? OptionalInt.of(value) : OptionalInt.empty(); - } - - public static List getBucketColumns(Map tableProperties) - { - return stringList(tableProperties.get(BUCKETED_ON_PROPERTY)); - } - - public static String getDistributionName(Map tableProperties) - { - return (String) tableProperties.get(DISTRIBUTION_NAME_PROPERTY); - } - - public static boolean isOrganized(Map tableProperties) - { - Boolean value = (Boolean) tableProperties.get(ORGANIZED_PROPERTY); - return (value == null) ? false : value; - } - - public static PropertyMetadata lowerCaseStringSessionProperty(String name, String description) - { - return new PropertyMetadata<>( - name, - description, - createUnboundedVarcharType(), - String.class, - null, - false, - value -> ((String) value).toLowerCase(ENGLISH), - value -> value); - } - - private static PropertyMetadata stringListSessionProperty(TypeManager typeManager, String name, String description) - { - return new PropertyMetadata<>( - name, - description, - typeManager.getParameterizedType(ARRAY, ImmutableList.of(TypeSignatureParameter.typeParameter(createUnboundedVarcharType().getTypeSignature()))), - List.class, - ImmutableList.of(), - false, - value -> stringList(value).stream() - .map(s -> s.toLowerCase(ENGLISH)) - .collect(toImmutableList()), - value -> value); - } - - @SuppressWarnings("unchecked") - private static List stringList(Object value) - { - return (value == null) ? ImmutableList.of() : ((List) value); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTransactionHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTransactionHandle.java deleted file mode 100644 index 66f058dc30cc..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTransactionHandle.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.connector.ConnectorTransactionHandle; - -import java.util.Objects; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class RaptorTransactionHandle - implements ConnectorTransactionHandle -{ - private final UUID uuid; - - public RaptorTransactionHandle() - { - this(UUID.randomUUID()); - } - - @JsonCreator - public RaptorTransactionHandle(@JsonProperty("uuid") UUID uuid) - { - this.uuid = requireNonNull(uuid, "uuid is null"); - } - - @JsonProperty - public UUID getUuid() - { - return uuid; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - RaptorTransactionHandle other = (RaptorTransactionHandle) obj; - return Objects.equals(uuid, other.uuid); - } - - @Override - public int hashCode() - { - return Objects.hash(uuid); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("uuid", uuid) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateFunction.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateFunction.java deleted file mode 100644 index e584d991ed65..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateFunction.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.airlift.slice.Slice; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.RowBlock; -import io.trino.spi.block.SqlRow; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.type.UuidType; - -public class RaptorUnbucketedUpdateFunction - implements BucketFunction -{ - private final int bucketCount; - - public RaptorUnbucketedUpdateFunction(int bucketCount) - { - this.bucketCount = bucketCount; - } - - @Override - public int getBucket(Page page, int position) - { - Block block = page.getBlock(0); - SqlRow row = ((RowBlock) block.getUnderlyingValueBlock()).getRow(block.getUnderlyingValuePosition(position)); - Slice uuid = UuidType.UUID.getSlice(row.getRawFieldBlock(1), row.getRawIndex()); // uuid field of row ID - return (uuid.hashCode() & Integer.MAX_VALUE) % bucketCount; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateHandle.java deleted file mode 100644 index 83e361b67fc1..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateHandle.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.spi.connector.ConnectorPartitioningHandle; - -public enum RaptorUnbucketedUpdateHandle - implements ConnectorPartitioningHandle -{ - INSTANCE -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupConfig.java deleted file mode 100644 index b081961f8f4e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupConfig.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; -import io.airlift.units.MaxDuration; -import io.airlift.units.MinDuration; -import jakarta.annotation.Nullable; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; - -import static java.util.concurrent.TimeUnit.MINUTES; - -public class BackupConfig -{ - private Duration timeout = new Duration(1, MINUTES); - private int timeoutThreads = 1000; - private String provider; - private int backupThreads = 5; - - @NotNull - @MinDuration("1s") - @MaxDuration("1h") - public Duration getTimeout() - { - return timeout; - } - - @Config("backup.timeout") - @ConfigDescription("Timeout for per-shard backup operations") - public BackupConfig setTimeout(Duration timeout) - { - this.timeout = timeout; - return this; - } - - @Min(1) - public int getTimeoutThreads() - { - return timeoutThreads; - } - - @Config("backup.timeout-threads") - @ConfigDescription("Maximum number of timeout threads for backup operations") - public BackupConfig setTimeoutThreads(int timeoutThreads) - { - this.timeoutThreads = timeoutThreads; - return this; - } - - @Nullable - public String getProvider() - { - return provider; - } - - @Config("backup.provider") - @ConfigDescription("Backup provider to use (supported types: file)") - public BackupConfig setProvider(String provider) - { - this.provider = provider; - return this; - } - - @Min(1) - public int getBackupThreads() - { - return backupThreads; - } - - @Config("backup.threads") - @ConfigDescription("Maximum number of shards to backup at once") - public BackupConfig setBackupThreads(int backupThreads) - { - this.backupThreads = backupThreads; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupManager.java deleted file mode 100644 index 258a78496f16..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupManager.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.io.Files; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.storage.BackupStats; -import io.trino.plugin.raptor.legacy.storage.StorageService; -import io.trino.spi.TrinoException; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Flatten; -import org.weakref.jmx.Managed; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.completedFuture; -import static java.util.concurrent.CompletableFuture.runAsync; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class BackupManager -{ - private static final Logger log = Logger.get(BackupManager.class); - - private final Optional backupStore; - private final StorageService storageService; - private final ExecutorService executorService; - - private final AtomicInteger pendingBackups = new AtomicInteger(); - private final BackupStats stats = new BackupStats(); - - @Inject - public BackupManager(Optional backupStore, StorageService storageService, BackupConfig config) - { - this(backupStore, storageService, config.getBackupThreads()); - } - - public BackupManager(Optional backupStore, StorageService storageService, int backupThreads) - { - checkArgument(backupThreads > 0, "backupThreads must be > 0"); - - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.storageService = requireNonNull(storageService, "storageService is null"); - this.executorService = newFixedThreadPool(backupThreads, daemonThreadsNamed("background-shard-backup-%s")); - } - - @PreDestroy - public void shutdown() - { - shutdownAndAwaitTermination(executorService, 10, SECONDS); - } - - public CompletableFuture submit(UUID uuid, File source) - { - requireNonNull(uuid, "uuid is null"); - requireNonNull(source, "source is null"); - - if (backupStore.isEmpty()) { - return completedFuture(null); - } - - // TODO: decrement when the running task is finished (not immediately on cancel) - pendingBackups.incrementAndGet(); - CompletableFuture future = runAsync(new BackgroundBackup(uuid, source), executorService); - future.whenComplete((none, throwable) -> pendingBackups.decrementAndGet()); - return future; - } - - private class BackgroundBackup - implements Runnable - { - private final UUID uuid; - private final File source; - private final long queuedTime = System.nanoTime(); - - public BackgroundBackup(UUID uuid, File source) - { - this.uuid = requireNonNull(uuid, "uuid is null"); - this.source = requireNonNull(source, "source is null"); - } - - @Override - public void run() - { - try { - stats.addQueuedTime(Duration.nanosSince(queuedTime)); - long start = System.nanoTime(); - - backupStore.get().backupShard(uuid, source); - stats.addCopyShardDataRate(DataSize.ofBytes(source.length()), Duration.nanosSince(start)); - - File restored = new File(storageService.getStagingFile(uuid) + ".validate"); - backupStore.get().restoreShard(uuid, restored); - - if (!filesEqual(source, restored)) { - stats.incrementBackupCorruption(); - - File quarantineBase = storageService.getQuarantineFile(uuid); - File quarantineOriginal = new File(quarantineBase.getPath() + ".original"); - File quarantineRestored = new File(quarantineBase.getPath() + ".restored"); - - log.error("Backup is corrupt after write. Quarantining local file: %s", quarantineBase); - if (!this.source.renameTo(quarantineOriginal) || !restored.renameTo(quarantineRestored)) { - log.warn("Quarantine of corrupt backup shard failed: %s", uuid); - } - - throw new TrinoException(RAPTOR_BACKUP_CORRUPTION, "Backup is corrupt after write: " + uuid); - } - - if (!restored.delete()) { - log.warn("Failed to delete staging file: %s", restored); - } - - stats.incrementBackupSuccess(); - } - catch (Throwable t) { - stats.incrementBackupFailure(); - throw t; - } - } - } - - @Managed - public int getPendingBackupCount() - { - return pendingBackups.get(); - } - - @Managed - @Flatten - public BackupStats getStats() - { - return stats; - } - - private static boolean filesEqual(File file1, File file2) - { - try { - return Files.equal(file1, file2); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupModule.java deleted file mode 100644 index 40847952a62b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupModule.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; -import com.google.inject.util.Providers; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.configuration.ConfigurationAwareModule; -import io.trino.spi.catalog.CatalogName; -import jakarta.annotation.Nullable; -import org.weakref.jmx.MBeanExporter; - -import java.util.Map; -import java.util.Optional; - -import static io.airlift.configuration.ConfigBinder.configBinder; - -public class BackupModule - extends AbstractConfigurationAwareModule -{ - private final Map providers; - - public BackupModule(Map providers) - { - this.providers = ImmutableMap.builder() - .put("file", new FileBackupModule()) - .put("http", new HttpBackupModule()) - .putAll(providers) - .buildOrThrow(); - } - - @Override - protected void setup(Binder binder) - { - configBinder(binder).bindConfig(BackupConfig.class); - - String provider = buildConfigObject(BackupConfig.class).getProvider(); - if (provider == null) { - binder.bind(BackupStore.class).toProvider(Providers.of(null)); - } - else { - Module module = providers.get(provider); - if (module == null) { - binder.addError("Unknown backup provider: %s", provider); - } - else if (module instanceof ConfigurationAwareModule) { - install(module); - } - else { - binder.install(module); - } - } - binder.bind(BackupService.class).to(BackupServiceManager.class).in(Scopes.SINGLETON); - } - - @Provides - @Singleton - private static Optional createBackupStore( - @Nullable BackupStore store, - LifeCycleManager lifeCycleManager, - MBeanExporter exporter, - CatalogName catalogName, - BackupConfig config) - { - if (store == null) { - return Optional.empty(); - } - - BackupStore proxy = new TimeoutBackupStore( - store, - catalogName.toString(), - config.getTimeout(), - config.getTimeoutThreads()); - - lifeCycleManager.addInstance(proxy); - - BackupStore managed = new ManagedBackupStore(proxy); - exporter.exportWithGeneratedName(managed, BackupStore.class, catalogName.toString()); - - return Optional.of(managed); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupOperationStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupOperationStats.java deleted file mode 100644 index 32d21d0d1488..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupOperationStats.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeStat; -import io.trino.spi.TrinoException; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.util.function.Supplier; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_NOT_FOUND; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_TIMEOUT; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class BackupOperationStats -{ - private final TimeStat time = new TimeStat(MILLISECONDS); - private final CounterStat successes = new CounterStat(); - private final CounterStat failures = new CounterStat(); - private final CounterStat timeouts = new CounterStat(); - - @Managed - @Nested - public TimeStat getTime() - { - return time; - } - - @Managed - @Nested - public CounterStat getSuccesses() - { - return successes; - } - - @Managed - @Nested - public CounterStat getFailures() - { - return failures; - } - - @Managed - @Nested - public CounterStat getTimeouts() - { - return timeouts; - } - - public void run(Runnable runnable) - { - run(() -> { - runnable.run(); - return null; - }); - } - - public V run(Supplier supplier) - { - try (TimeStat.BlockTimer _ = time.time()) { - V value = supplier.get(); - successes.update(1); - return value; - } - catch (TrinoException e) { - if (e.getErrorCode().equals(RAPTOR_BACKUP_NOT_FOUND.toErrorCode())) { - successes.update(1); - } - else if (e.getErrorCode().equals(RAPTOR_BACKUP_TIMEOUT.toErrorCode())) { - timeouts.update(1); - } - else { - failures.update(1); - } - throw e; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupService.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupService.java deleted file mode 100644 index 5d8a66f3b26d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupService.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -public interface BackupService -{ - boolean isBackupAvailable(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupServiceManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupServiceManager.java deleted file mode 100644 index 7eef51dbd7f0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupServiceManager.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.inject.Inject; - -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public class BackupServiceManager - implements BackupService -{ - private final Optional backupStore; - - @Inject - public BackupServiceManager(Optional backupStore) - { - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - } - - @Override - public boolean isBackupAvailable() - { - return backupStore.isPresent(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupStore.java deleted file mode 100644 index dff2ab3637be..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupStore.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import java.io.File; -import java.util.UUID; - -public interface BackupStore -{ - /** - * Backup a shard by copying it to the backup store. - * - * @param uuid shard UUID - * @param source the source file - */ - void backupShard(UUID uuid, File source); - - /** - * Restore a shard by copying it from the backup store. - * - * @param uuid shard UUID - * @param target the destination file - */ - void restoreShard(UUID uuid, File target); - - /** - * Delete shard from the backup store if it exists. - * - * @param uuid shard UUID - * @return {@code true} if the shard was deleted; {@code false} if it did not exist - */ - boolean deleteShard(UUID uuid); - - /** - * Check if a shard exists in the backup store. - * - * @param uuid shard UUID - * @return if the shard exists - */ - boolean shardExists(UUID uuid); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupConfig.java deleted file mode 100644 index 7f54dc56ce44..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import jakarta.validation.constraints.NotNull; - -import java.io.File; - -public class FileBackupConfig -{ - private File backupDirectory; - - @NotNull - public File getBackupDirectory() - { - return backupDirectory; - } - - @Config("backup.directory") - @ConfigDescription("Base directory to use for the backup copy of shard data") - public FileBackupConfig setBackupDirectory(File backupDirectory) - { - this.backupDirectory = backupDirectory; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupModule.java deleted file mode 100644 index 604574c1605c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupModule.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Scopes; - -import static io.airlift.configuration.ConfigBinder.configBinder; - -public class FileBackupModule - implements Module -{ - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(FileBackupConfig.class); - binder.bind(BackupStore.class).to(FileBackupStore.class).in(Scopes.SINGLETON); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupStore.java deleted file mode 100644 index 33943038ea91..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupStore.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; -import io.trino.spi.TrinoException; -import jakarta.annotation.PostConstruct; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.UUID; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_NOT_FOUND; -import static io.trino.plugin.raptor.legacy.storage.FileStorageService.getFileSystemPath; -import static java.nio.file.Files.deleteIfExists; -import static java.util.Objects.requireNonNull; - -public class FileBackupStore - implements BackupStore -{ - private final File baseDir; - - @Inject - public FileBackupStore(FileBackupConfig config) - { - this(config.getBackupDirectory()); - } - - public FileBackupStore(File baseDir) - { - this.baseDir = requireNonNull(baseDir, "baseDir is null"); - } - - @PostConstruct - public void start() - { - createDirectories(baseDir); - } - - @Override - public void backupShard(UUID uuid, File source) - { - File backupFile = getBackupFile(uuid); - - try { - try { - // Optimistically assume the file can be created - copyFile(source, backupFile); - } - catch (FileNotFoundException e) { - createDirectories(backupFile.getParentFile()); - copyFile(source, backupFile); - } - } - catch (IOException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to create backup shard file", e); - } - } - - @Override - public void restoreShard(UUID uuid, File target) - { - try { - copyFile(getBackupFile(uuid), target); - } - catch (FileNotFoundException e) { - throw new TrinoException(RAPTOR_BACKUP_NOT_FOUND, "Backup shard not found: " + uuid, e); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to copy backup shard: " + uuid, e); - } - } - - @Override - public boolean deleteShard(UUID uuid) - { - try { - return deleteIfExists(getBackupFile(uuid).toPath()); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to delete backup shard: " + uuid, e); - } - } - - @Override - public boolean shardExists(UUID uuid) - { - return getBackupFile(uuid).isFile(); - } - - @VisibleForTesting - public File getBackupFile(UUID uuid) - { - return getFileSystemPath(baseDir, uuid); - } - - private static void createDirectories(File dir) - { - if (!dir.mkdirs() && !dir.isDirectory()) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed creating directories: " + dir); - } - } - - private static void copyFile(File source, File target) - throws IOException - { - try (InputStream in = new FileInputStream(source); - FileOutputStream out = new FileOutputStream(target)) { - in.transferTo(out); - out.flush(); - out.getFD().sync(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ForHttpBackup.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ForHttpBackup.java deleted file mode 100644 index 7f62f6a0a2b3..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ForHttpBackup.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.inject.BindingAnnotation; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -@Retention(RUNTIME) -@Target({FIELD, PARAMETER, METHOD}) -@BindingAnnotation -public @interface ForHttpBackup -{ -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupConfig.java deleted file mode 100644 index 2dfceae6015a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import jakarta.validation.constraints.NotNull; - -import java.net.URI; - -public class HttpBackupConfig -{ - private URI uri; - - @NotNull - public URI getUri() - { - return uri; - } - - @Config("backup.http.uri") - @ConfigDescription("Backup service base URI") - public HttpBackupConfig setUri(URI uri) - { - this.uri = uri; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupModule.java deleted file mode 100644 index 739bc9fa815b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupModule.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; - -import java.net.URI; -import java.util.function.Supplier; - -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.http.client.HttpClientBinder.httpClientBinder; - -public class HttpBackupModule - implements Module -{ - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(HttpBackupConfig.class); - binder.bind(BackupStore.class).to(HttpBackupStore.class).in(Scopes.SINGLETON); - - httpClientBinder(binder).bindHttpClient("backup", ForHttpBackup.class); - } - - @Provides - @Singleton - @ForHttpBackup - public Supplier createBackupUriSupplier(HttpBackupConfig config) - { - URI uri = config.getUri(); - return () -> uri; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupStore.java deleted file mode 100644 index cac0405f8869..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupStore.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.io.ByteStreams; -import com.google.inject.Inject; -import io.airlift.http.client.FileBodyGenerator; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; -import io.airlift.http.client.StatusResponseHandler.StatusResponse; -import io.airlift.slice.XxHash64; -import io.trino.spi.NodeManager; -import io.trino.spi.TrinoException; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.UUID; -import java.util.function.Supplier; - -import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static com.google.common.net.MediaType.APPLICATION_BINARY; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.Request.Builder.prepareHead; -import static io.airlift.http.client.Request.Builder.preparePut; -import static io.airlift.http.client.ResponseHandlerUtils.propagate; -import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_ERROR; -import static java.lang.String.format; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.requireNonNull; - -public class HttpBackupStore - implements BackupStore -{ - public static final String TRINO_ENVIRONMENT = "X-Trino-Environment"; - public static final String CONTENT_XXH64 = "X-Content-XXH64"; - - private final HttpClient httpClient; - private final Supplier baseUriSupplier; - private final String environment; - - @Inject - public HttpBackupStore( - @ForHttpBackup HttpClient httpClient, - @ForHttpBackup Supplier baseUriSupplier, - NodeManager nodeManager) - { - this.httpClient = requireNonNull(httpClient, "httpClient is null"); - this.baseUriSupplier = requireNonNull(baseUriSupplier, "baseUriSupplier is null"); - this.environment = nodeManager.getEnvironment(); - } - - @Override - public void backupShard(UUID uuid, File source) - { - Request request = preparePut() - .addHeader(TRINO_ENVIRONMENT, environment) - .addHeader(CONTENT_TYPE, APPLICATION_BINARY.toString()) - .addHeader(CONTENT_XXH64, format("%016x", xxHash64(source))) - .setUri(shardUri(uuid)) - .setBodyGenerator(new FileBodyGenerator(source.toPath())) - .build(); - - try { - StatusResponse status = httpClient.execute(request, createStatusResponseHandler()); - if (!isOk(status)) { - throw badResponse(status); - } - } - catch (RuntimeException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to backup shard: " + uuid, e); - } - } - - @Override - public void restoreShard(UUID uuid, File target) - { - Request request = prepareGet() - .addHeader(TRINO_ENVIRONMENT, environment) - .setUri(shardUri(uuid)) - .build(); - - try { - StatusResponse status = httpClient.execute(request, new FileResponseHandler(target)); - if (isNotFound(status) || isGone(status)) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Backup shard not found: " + uuid); - } - if (!isOk(status)) { - throw badResponse(status); - } - } - catch (IOException | RuntimeException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to restore shard: " + uuid, e); - } - } - - @Override - public boolean deleteShard(UUID uuid) - { - Request request = prepareDelete() - .addHeader(TRINO_ENVIRONMENT, environment) - .setUri(shardUri(uuid)) - .build(); - - try { - StatusResponse status = httpClient.execute(request, createStatusResponseHandler()); - if (isOk(status) || isGone(status)) { - return true; - } - if (isNotFound(status)) { - return false; - } - throw badResponse(status); - } - catch (RuntimeException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to delete shard: " + uuid, e); - } - } - - @Override - public boolean shardExists(UUID uuid) - { - Request request = prepareHead() - .addHeader(TRINO_ENVIRONMENT, environment) - .setUri(shardUri(uuid)) - .build(); - - try { - StatusResponse status = httpClient.execute(request, createStatusResponseHandler()); - if (isOk(status)) { - return true; - } - if (isNotFound(status) || isGone(status)) { - return false; - } - throw badResponse(status); - } - catch (RuntimeException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to check if shard exists: " + uuid, e); - } - } - - private URI shardUri(UUID uuid) - { - return uriBuilderFrom(baseUriSupplier.get()) - .appendPath(uuid.toString().toLowerCase(ENGLISH)) - .build(); - } - - private static boolean isOk(StatusResponse response) - { - return (response.getStatusCode() == HttpStatus.OK.code()) || - (response.getStatusCode() == HttpStatus.NO_CONTENT.code()); - } - - private static boolean isNotFound(StatusResponse response) - { - return response.getStatusCode() == HttpStatus.NOT_FOUND.code(); - } - - private static boolean isGone(StatusResponse response) - { - return response.getStatusCode() == HttpStatus.GONE.code(); - } - - private static RuntimeException badResponse(StatusResponse response) - { - throw new RuntimeException("Request failed with HTTP status " + response.getStatusCode()); - } - - private static long xxHash64(File file) - { - try (InputStream in = new FileInputStream(file)) { - return XxHash64.hash(in); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to read file: " + file, e); - } - } - - private static class FileResponseHandler - implements ResponseHandler - { - private final File file; - - private FileResponseHandler(File file) - { - this.file = requireNonNull(file, "file is null"); - } - - @Override - public StatusResponse handleException(Request request, Exception exception) - { - throw propagate(request, exception); - } - - @Override - public StatusResponse handle(Request request, Response response) - throws IOException - { - StatusResponse status = createStatusResponse(response); - if (isOk(status)) { - writeFile(response.getInputStream()); - } - return status; - } - - private void writeFile(InputStream in) - throws IOException - { - try (FileOutputStream out = new FileOutputStream(file)) { - ByteStreams.copy(in, out); - out.flush(); - out.getFD().sync(); - } - } - - private static StatusResponse createStatusResponse(Response response) - { - return new StatusResponse(response.getStatusCode(), response.getHeaders()); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ManagedBackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ManagedBackupStore.java deleted file mode 100644 index 45b31e75681c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ManagedBackupStore.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.log.Logger; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.io.File; -import java.util.UUID; - -import static java.util.Objects.requireNonNull; - -public class ManagedBackupStore - implements BackupStore -{ - private final BackupStore store; - private final Logger log; - - private final BackupOperationStats backupShard = new BackupOperationStats(); - private final BackupOperationStats restoreShard = new BackupOperationStats(); - private final BackupOperationStats deleteShard = new BackupOperationStats(); - private final BackupOperationStats shardExists = new BackupOperationStats(); - - public ManagedBackupStore(BackupStore store) - { - this.store = requireNonNull(store, "store is null"); - this.log = Logger.get(store.getClass()); - } - - @Override - public void backupShard(UUID uuid, File source) - { - log.debug("Creating shard backup: %s", uuid); - backupShard.run(() -> store.backupShard(uuid, source)); - } - - @Override - public void restoreShard(UUID uuid, File target) - { - log.debug("Restoring shard backup: %s", uuid); - restoreShard.run(() -> store.restoreShard(uuid, target)); - } - - @Override - public boolean deleteShard(UUID uuid) - { - log.debug("Deleting shard backup: %s", uuid); - return deleteShard.run(() -> store.deleteShard(uuid)); - } - - @Override - public boolean shardExists(UUID uuid) - { - return shardExists.run(() -> store.shardExists(uuid)); - } - - @Managed - @Nested - public BackupOperationStats getBackupShard() - { - return backupShard; - } - - @Managed - @Nested - public BackupOperationStats getRestoreShard() - { - return restoreShard; - } - - @Managed - @Nested - public BackupOperationStats getDeleteShard() - { - return deleteShard; - } - - @Managed - @Nested - public BackupOperationStats getShardExists() - { - return shardExists; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/TimeoutBackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/TimeoutBackupStore.java deleted file mode 100644 index f627b6bf369e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/TimeoutBackupStore.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.util.concurrent.SimpleTimeLimiter; -import com.google.common.util.concurrent.TimeLimiter; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.concurrent.ExecutorServiceAdapter; -import io.airlift.units.Duration; -import io.trino.spi.TrinoException; -import jakarta.annotation.PreDestroy; - -import java.io.File; -import java.util.UUID; -import java.util.concurrent.ExecutorService; - -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_TIMEOUT; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class TimeoutBackupStore - implements BackupStore -{ - private final ExecutorService executor; - private final BackupStore store; - - public TimeoutBackupStore(BackupStore store, String connectorId, Duration timeout, int maxThreads) - { - requireNonNull(store, "store is null"); - requireNonNull(connectorId, "connectorId is null"); - requireNonNull(timeout, "timeout is null"); - - this.executor = newCachedThreadPool(daemonThreadsNamed("backup-proxy-" + connectorId + "-%s")); - this.store = timeLimited(store, BackupStore.class, timeout, executor, maxThreads); - } - - @PreDestroy - public void shutdown() - { - executor.shutdownNow(); - } - - @Override - public void backupShard(UUID uuid, File source) - { - try { - store.backupShard(uuid, source); - } - catch (UncheckedTimeoutException e) { - timeoutException(uuid, "Shard backup timed out"); - } - } - - @Override - public void restoreShard(UUID uuid, File target) - { - try { - store.restoreShard(uuid, target); - } - catch (UncheckedTimeoutException e) { - timeoutException(uuid, "Shard restore timed out"); - } - } - - @Override - public boolean deleteShard(UUID uuid) - { - try { - return store.deleteShard(uuid); - } - catch (UncheckedTimeoutException e) { - throw timeoutException(uuid, "Shard delete timed out"); - } - } - - @Override - public boolean shardExists(UUID uuid) - { - try { - return store.shardExists(uuid); - } - catch (UncheckedTimeoutException e) { - throw timeoutException(uuid, "Shard existence check timed out"); - } - } - - private static T timeLimited(T target, Class clazz, Duration timeout, ExecutorService executor, int maxThreads) - { - executor = new ExecutorServiceAdapter(new BoundedExecutor(executor, maxThreads)); - TimeLimiter limiter = SimpleTimeLimiter.create(executor); - return limiter.newProxy(target, clazz, timeout.toMillis(), MILLISECONDS); - } - - private static TrinoException timeoutException(UUID uuid, String message) - { - throw new TrinoException(RAPTOR_BACKUP_TIMEOUT, message + ": " + uuid); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/AssignmentLimiter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/AssignmentLimiter.java deleted file mode 100644 index fedc32a774c2..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/AssignmentLimiter.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.base.Ticker; -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.spi.Node; -import io.trino.spi.TrinoException; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.OptionalLong; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; - -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_REASSIGNMENT_DELAY; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_REASSIGNMENT_THROTTLE; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toSet; - -public class AssignmentLimiter -{ - private static final Logger log = Logger.get(AssignmentLimiter.class); - - private final NodeSupplier nodeSupplier; - private final Ticker ticker; - private final Duration reassignmentDelay; - private final Duration reassignmentInterval; - - private final ScheduledExecutorService scheduler = newScheduledThreadPool(1, daemonThreadsNamed("assignment-limiter")); - private final AtomicBoolean started = new AtomicBoolean(); - - @GuardedBy("this") - private final Map delayedNodes = new HashMap<>(); - @GuardedBy("this") - private final Set offlineNodes = new HashSet<>(); - @GuardedBy("this") - private OptionalLong lastOfflined = OptionalLong.empty(); - - @Inject - public AssignmentLimiter(NodeSupplier nodeSupplier, Ticker ticker, MetadataConfig config) - { - this(nodeSupplier, ticker, config.getReassignmentDelay(), config.getReassignmentInterval()); - } - - public AssignmentLimiter(NodeSupplier nodeSupplier, Ticker ticker, Duration reassignmentDelay, Duration reassignmentInterval) - { - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.ticker = requireNonNull(ticker, "ticker is null"); - this.reassignmentDelay = requireNonNull(reassignmentDelay, "reassignmentDelay is null"); - this.reassignmentInterval = requireNonNull(reassignmentInterval, "reassignmentInterval is null"); - } - - @PostConstruct - public void start() - { - if (!started.getAndSet(true)) { - scheduler.scheduleWithFixedDelay(() -> { - try { - clearOnlineNodes(); - } - catch (Throwable t) { - log.error(t, "Error clearing online nodes"); - } - }, 2, 2, SECONDS); - } - } - - @PreDestroy - public void shutdown() - { - scheduler.shutdownNow(); - } - - public synchronized void checkAssignFrom(String nodeIdentifier) - { - if (offlineNodes.contains(nodeIdentifier)) { - return; - } - - long now = ticker.read(); - long start = delayedNodes.computeIfAbsent(nodeIdentifier, key -> now); - Duration delay = new Duration(now - start, NANOSECONDS); - - if (delay.compareTo(reassignmentDelay) < 0) { - throw new TrinoException(RAPTOR_REASSIGNMENT_DELAY, format( - "Reassignment delay is in effect for node %s (elapsed: %s)", - nodeIdentifier, - delay.convertToMostSuccinctTimeUnit())); - } - - if (lastOfflined.isPresent()) { - delay = new Duration(now - lastOfflined.getAsLong(), NANOSECONDS); - if (delay.compareTo(reassignmentInterval) < 0) { - throw new TrinoException(RAPTOR_REASSIGNMENT_THROTTLE, format( - "Reassignment throttle is in effect for node %s (elapsed: %s)", - nodeIdentifier, - delay.convertToMostSuccinctTimeUnit())); - } - } - - delayedNodes.remove(nodeIdentifier); - offlineNodes.add(nodeIdentifier); - lastOfflined = OptionalLong.of(now); - } - - private void clearOnlineNodes() - { - Set onlineNodes = nodeSupplier.getWorkerNodes().stream() - .map(Node::getNodeIdentifier) - .collect(toSet()); - - synchronized (this) { - delayedNodes.keySet().removeAll(onlineNodes); - offlineNodes.removeAll(onlineNodes); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketNode.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketNode.java deleted file mode 100644 index 0d329b0b9f37..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketNode.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import java.util.Objects; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class BucketNode -{ - private final int bucketNumber; - private final String nodeIdentifier; - - public BucketNode(int bucketNumber, String nodeIdentifier) - { - checkArgument(bucketNumber >= 0, "bucket number must be positive"); - this.bucketNumber = bucketNumber; - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - } - - public int getBucketNumber() - { - return bucketNumber; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if ((o == null) || (getClass() != o.getClass())) { - return false; - } - BucketNode that = (BucketNode) o; - return (bucketNumber == that.bucketNumber) && - Objects.equals(nodeIdentifier, that.nodeIdentifier); - } - - @Override - public int hashCode() - { - return Objects.hash(bucketNumber, nodeIdentifier); - } - - @Override - public String toString() - { - return bucketNumber + ":" + nodeIdentifier; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketReassigner.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketReassigner.java deleted file mode 100644 index 680c40a8c112..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketReassigner.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ThreadLocalRandom; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class BucketReassigner -{ - private final Map nodeBucketCounts = new HashMap<>(); - private final List activeNodes; - private final List bucketNodes; - private boolean initialized; - - public BucketReassigner(Set activeNodes, List bucketNodes) - { - checkArgument(!activeNodes.isEmpty(), "activeNodes must not be empty"); - this.activeNodes = ImmutableList.copyOf(requireNonNull(activeNodes, "activeNodes is null")); - this.bucketNodes = requireNonNull(bucketNodes, "bucketNodes is null"); - } - - // NOTE: This method is not thread safe - public String getNextReassignmentDestination() - { - if (!initialized) { - for (String node : activeNodes) { - nodeBucketCounts.put(node, 0); - } - for (BucketNode bucketNode : bucketNodes) { - nodeBucketCounts.computeIfPresent(bucketNode.getNodeIdentifier(), (node, bucketCount) -> bucketCount + 1); - } - initialized = true; - } - - String assignedNode; - if (activeNodes.size() > 1) { - assignedNode = randomTwoChoices(); - } - else { - assignedNode = activeNodes.get(0); - } - - nodeBucketCounts.compute(assignedNode, (node, count) -> count + 1); - return assignedNode; - } - - private String randomTwoChoices() - { - // Purely random choices can overload unlucky node while selecting the least loaded one based on stale - // local information can overload the previous idle node. Here we randomly pick 2 nodes and select the - // less loaded one. This prevents those issues and renders good enough load balance. - int randomPosition = ThreadLocalRandom.current().nextInt(activeNodes.size()); - int randomOffset = ThreadLocalRandom.current().nextInt(1, activeNodes.size()); - String candidate1 = activeNodes.get(randomPosition); - String candidate2 = activeNodes.get((randomPosition + randomOffset) % activeNodes.size()); - - return (nodeBucketCounts.get(candidate1) <= nodeBucketCounts.get(candidate2)) ? candidate1 : candidate2; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketShards.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketShards.java deleted file mode 100644 index d38f1efe0424..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketShards.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableSet; - -import java.util.Objects; -import java.util.OptionalInt; -import java.util.Set; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class BucketShards -{ - private final OptionalInt bucketNumber; - private final Set shards; - - public BucketShards(OptionalInt bucketNumber, Set shards) - { - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.shards = ImmutableSet.copyOf(requireNonNull(shards, "shards is null")); - } - - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - public Set getShards() - { - return shards; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - BucketShards other = (BucketShards) obj; - return Objects.equals(this.bucketNumber, other.bucketNumber) && - Objects.equals(this.shards, other.shards); - } - - @Override - public int hashCode() - { - return Objects.hash(bucketNumber, shards); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .add("shards", shards) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnInfo.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnInfo.java deleted file mode 100644 index d4f6b36cce8f..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnInfo.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.type.Type; - -import static java.util.Objects.requireNonNull; - -public class ColumnInfo -{ - private final long columnId; - private final Type type; - - public ColumnInfo(long columnId, Type type) - { - this.columnId = columnId; - this.type = requireNonNull(type, "type is null"); - } - - public long getColumnId() - { - return columnId; - } - - public Type getType() - { - return type; - } - - @Override - public String toString() - { - return columnId + ":" + type; - } - - public static ColumnInfo fromHandle(RaptorColumnHandle handle) - { - return new ColumnInfo(handle.getColumnId(), handle.getColumnType()); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnMetadataRow.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnMetadataRow.java deleted file mode 100644 index 0d5077f19b8f..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnMetadataRow.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import java.util.OptionalInt; - -import static java.util.Objects.requireNonNull; - -public class ColumnMetadataRow -{ - private final long tableId; - private final long columnId; - private final String columnName; - private final OptionalInt sortOrdinalPosition; - private final OptionalInt bucketOrdinalPosition; - - public ColumnMetadataRow(long tableId, long columnId, String columnName, OptionalInt sortOrdinalPosition, OptionalInt bucketOrdinalPosition) - { - this.tableId = tableId; - this.columnId = columnId; - this.columnName = requireNonNull(columnName, "columnName is null"); - this.sortOrdinalPosition = requireNonNull(sortOrdinalPosition, "sortOrdinalPosition is null"); - this.bucketOrdinalPosition = requireNonNull(bucketOrdinalPosition, "bucketOrdinalPosition is null"); - } - - public long getTableId() - { - return tableId; - } - - public long getColumnId() - { - return columnId; - } - - public String getColumnName() - { - return columnName; - } - - public OptionalInt getSortOrdinalPosition() - { - return sortOrdinalPosition; - } - - public OptionalInt getBucketOrdinalPosition() - { - return bucketOrdinalPosition; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnStats.java deleted file mode 100644 index 77b63ffbfe4d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnStats.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.annotation.Nullable; - -import static com.google.common.base.MoreObjects.toStringHelper; - -public class ColumnStats -{ - private final long columnId; - private final Object min; - private final Object max; - - @JsonCreator - public ColumnStats( - @JsonProperty("columnId") long columnId, - @JsonProperty("min") @Nullable Object min, - @JsonProperty("max") @Nullable Object max) - { - this.columnId = columnId; - this.min = min; - this.max = max; - } - - @JsonProperty - public long getColumnId() - { - return columnId; - } - - @Nullable - @JsonProperty - public Object getMin() - { - return min; - } - - @Nullable - @JsonProperty - public Object getMax() - { - return max; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("columnId", columnId) - .add("min", min) - .add("max", max) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseConfig.java deleted file mode 100644 index 26befd10a601..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import jakarta.validation.constraints.NotNull; - -public class DatabaseConfig -{ - private String databaseType; - - @NotNull - public String getDatabaseType() - { - return databaseType; - } - - @Config("metadata.db.type") - @ConfigDescription("Metadata database type (supported types: h2, mysql)") - public DatabaseConfig setDatabaseType(String databaseType) - { - this.databaseType = databaseType; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseMetadataModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseMetadataModule.java deleted file mode 100644 index 4cf735884e1d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseMetadataModule.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import org.jdbi.v3.core.ConnectionFactory; -import org.jdbi.v3.core.Jdbi; - -import java.sql.DriverManager; - -import static io.airlift.configuration.ConditionalModule.conditionalModule; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static java.lang.String.format; - -public class DatabaseMetadataModule - extends AbstractConfigurationAwareModule -{ - @Override - protected void setup(Binder ignored) - { - install(conditionalModule( - DatabaseConfig.class, - config -> "mysql".equals(config.getDatabaseType()), - new MySqlModule())); - - install(conditionalModule( - DatabaseConfig.class, - config -> "h2".equals(config.getDatabaseType()), - new H2Module())); - } - - private static class MySqlModule - implements Module - { - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(JdbcDatabaseConfig.class); - } - - @Provides - @Singleton - @ForMetadata - public static ConnectionFactory createConnectionFactory(JdbcDatabaseConfig config) - { - String url = config.getUrl(); - return () -> DriverManager.getConnection(url); - } - - @Provides - @Singleton - public static DaoSupplier createShardDaoSupplier(@ForMetadata Jdbi dbi) - { - return new DaoSupplier<>(dbi, MySqlShardDao.class); - } - } - - private static class H2Module - implements Module - { - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(H2DatabaseConfig.class); - } - - @Provides - @Singleton - @ForMetadata - public static ConnectionFactory createConnectionFactory(H2DatabaseConfig config) - { - String url = format("jdbc:h2:%s;DB_CLOSE_DELAY=-1", config.getFilename()); - return () -> DriverManager.getConnection(url); - } - - @Provides - @Singleton - public static DaoSupplier createShardDaoSupplier(@ForMetadata Jdbi dbi) - { - return new DaoSupplier<>(dbi, H2ShardDao.class); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardManager.java deleted file mode 100644 index 447981fcf0e0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardManager.java +++ /dev/null @@ -1,933 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.base.Joiner; -import com.google.common.base.Ticker; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.UncheckedExecutionException; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.cache.NonEvictableLoadingCache; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerDao; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.Node; -import io.trino.spi.TrinoException; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import org.h2.jdbc.JdbcConnection; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.HandleConsumer; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.JdbiException; -import org.jdbi.v3.core.result.ResultIterator; - -import java.sql.Connection; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.StringJoiner; -import java.util.UUID; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Throwables.throwIfInstanceOf; -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.Iterables.partition; -import static io.trino.cache.SafeCaches.buildNonEvictableCache; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS; -import static io.trino.plugin.raptor.legacy.storage.ColumnIndexStatsUtils.jdbcType; -import static io.trino.plugin.raptor.legacy.storage.ShardStats.MAX_BINARY_INDEX_SIZE; -import static io.trino.plugin.raptor.legacy.util.ArrayUtil.intArrayFromBytes; -import static io.trino.plugin.raptor.legacy.util.ArrayUtil.intArrayToBytes; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.bindOptionalInt; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.isSyntaxOrAccessError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.isTransactionCacheFullError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runIgnoringConstraintViolation; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runTransaction; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidToBytes; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static io.trino.spi.StandardErrorCode.NO_NODES_AVAILABLE; -import static io.trino.spi.StandardErrorCode.SERVER_STARTING_UP; -import static io.trino.spi.StandardErrorCode.TRANSACTION_CONFLICT; -import static java.lang.Boolean.TRUE; -import static java.lang.Math.multiplyExact; -import static java.lang.String.format; -import static java.sql.Statement.RETURN_GENERATED_KEYS; -import static java.util.Arrays.asList; -import static java.util.Collections.nCopies; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - -public class DatabaseShardManager - implements ShardManager -{ - private static final Logger log = Logger.get(DatabaseShardManager.class); - - private static final String INDEX_TABLE_PREFIX = "x_shards_t"; - private static final int MAX_ADD_COLUMN_ATTEMPTS = 100; - - private final Jdbi dbi; - private final DaoSupplier shardDaoSupplier; - private final ShardDao dao; - private final NodeSupplier nodeSupplier; - private final AssignmentLimiter assignmentLimiter; - private final Ticker ticker; - private final Duration startupGracePeriod; - private final long startTime; - - private final NonEvictableLoadingCache nodeIdCache = buildNonEvictableCache( - CacheBuilder.newBuilder().maximumSize(10_000), - CacheLoader.from(this::loadNodeId)); - - private final NonEvictableLoadingCache> bucketAssignmentsCache = buildNonEvictableCache( - CacheBuilder.newBuilder().expireAfterWrite(1, SECONDS), - CacheLoader.from(this::loadBucketAssignments)); - - @Inject - public DatabaseShardManager( - @ForMetadata Jdbi dbi, - DaoSupplier shardDaoSupplier, - NodeSupplier nodeSupplier, - AssignmentLimiter assignmentLimiter, - Ticker ticker, - MetadataConfig config) - { - this(dbi, shardDaoSupplier, nodeSupplier, assignmentLimiter, ticker, config.getStartupGracePeriod()); - } - - public DatabaseShardManager( - Jdbi dbi, - DaoSupplier shardDaoSupplier, - NodeSupplier nodeSupplier, - AssignmentLimiter assignmentLimiter, - Ticker ticker, - Duration startupGracePeriod) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.shardDaoSupplier = requireNonNull(shardDaoSupplier, "shardDaoSupplier is null"); - this.dao = shardDaoSupplier.onDemand(); - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.assignmentLimiter = requireNonNull(assignmentLimiter, "assignmentLimiter is null"); - this.ticker = requireNonNull(ticker, "ticker is null"); - this.startupGracePeriod = requireNonNull(startupGracePeriod, "startupGracePeriod is null"); - this.startTime = ticker.read(); - } - - @Override - public void createTable(long tableId, List columns, boolean bucketed, OptionalLong temporalColumnId) - { - StringJoiner tableColumns = new StringJoiner(",\n ", " ", ",\n").setEmptyValue(""); - - for (ColumnInfo column : columns) { - String columnType = sqlColumnType(column.getType()); - if (columnType != null) { - tableColumns.add(minColumn(column.getColumnId()) + " " + columnType); - tableColumns.add(maxColumn(column.getColumnId()) + " " + columnType); - } - } - - StringJoiner coveringIndexColumns = new StringJoiner(", "); - - // Add the max temporal column first to accelerate queries that usually scan recent data - temporalColumnId.ifPresent(id -> coveringIndexColumns.add(maxColumn(id))); - temporalColumnId.ifPresent(id -> coveringIndexColumns.add(minColumn(id))); - - String sql; - if (bucketed) { - coveringIndexColumns - .add("bucket_number") - .add("shard_id") - .add("shard_uuid"); - - sql = "" + - "CREATE TABLE " + shardIndexTable(tableId) + " (\n" + - " shard_id BIGINT NOT NULL,\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " bucket_number INT NOT NULL\n," + - tableColumns + - " PRIMARY KEY (bucket_number, shard_uuid),\n" + - " UNIQUE (shard_id),\n" + - " UNIQUE (shard_uuid),\n" + - " UNIQUE (" + coveringIndexColumns + ")\n" + - ")"; - } - else { - coveringIndexColumns - .add("node_ids") - .add("shard_id") - .add("shard_uuid"); - - sql = "" + - "CREATE TABLE " + shardIndexTable(tableId) + " (\n" + - " shard_id BIGINT NOT NULL,\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " node_ids VARBINARY(128) NOT NULL,\n" + - tableColumns + - " PRIMARY KEY (node_ids, shard_uuid),\n" + - " UNIQUE (shard_id),\n" + - " UNIQUE (shard_uuid),\n" + - " UNIQUE (" + coveringIndexColumns + ")\n" + - ")"; - } - - try (Handle handle = dbi.open()) { - handle.execute(sql); - } - catch (JdbiException e) { - throw metadataError(e); - } - } - - @Override - public void dropTable(long tableId) - { - runTransaction(dbi, handle -> { - lockTable(handle, tableId); - - ShardDao shardDao = shardDaoSupplier.attach(handle); - shardDao.insertDeletedShards(tableId); - shardDao.dropShardNodes(tableId); - shardDao.dropShards(tableId); - - handle.attach(ShardOrganizerDao.class).dropOrganizerJobs(tableId); - - MetadataDao dao = handle.attach(MetadataDao.class); - dao.dropColumns(tableId); - dao.dropTable(tableId); - return null; - }); - - // TODO: add a cleanup process for leftover index tables - // It is not possible to drop the index tables in a transaction. - try (Handle handle = dbi.open()) { - handle.execute("DROP TABLE " + shardIndexTable(tableId)); - } - catch (JdbiException e) { - log.warn(e, "Failed to drop index table %s", shardIndexTable(tableId)); - } - } - - @Override - public void addColumn(long tableId, ColumnInfo column) - { - String columnType = sqlColumnType(column.getType()); - if (columnType == null) { - // TODO we should probably fail here - return; - } - - String sql = format("ALTER TABLE %s ADD COLUMN (%s %s, %s %s)", - shardIndexTable(tableId), - minColumn(column.getColumnId()), columnType, - maxColumn(column.getColumnId()), columnType); - - int attempts = 0; - while (true) { - attempts++; - try (Handle handle = dbi.open()) { - handle.execute(sql); - } - catch (JdbiException e) { - if (isSyntaxOrAccessError(e)) { - // exit when column already exists - return; - } - if (attempts >= MAX_ADD_COLUMN_ATTEMPTS) { - throw metadataError(e); - } - log.warn(e, "Failed to alter table on attempt %s, will retry. SQL: %s", attempts, sql); - try { - SECONDS.sleep(3); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw metadataError(ie); - } - } - } - } - - @Override - public void commitShards(long transactionId, long tableId, List columns, Collection shards, Optional externalBatchId, long updateTime) - { - // attempt to fail up front with a proper exception - if (externalBatchId.isPresent() && dao.externalBatchExists(externalBatchId.get())) { - throw new TrinoException(RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS, "External batch already exists: " + externalBatchId.get()); - } - - Map nodeIds = toNodeIdMap(shards); - - runCommit(transactionId, handle -> { - externalBatchId.ifPresent(shardDaoSupplier.attach(handle)::insertExternalBatch); - lockTable(handle, tableId); - insertShardsAndIndex(tableId, columns, shards, nodeIds, handle); - - ShardStats stats = shardStats(shards); - MetadataDao metadata = handle.attach(MetadataDao.class); - metadata.updateTableStats(tableId, shards.size(), stats.getRowCount(), stats.getCompressedSize(), stats.getUncompressedSize()); - metadata.updateTableVersion(tableId, updateTime); - }); - } - - @Override - public void replaceShardUuids(long transactionId, long tableId, List columns, Set oldShardUuids, Collection newShards, OptionalLong updateTime) - { - Map nodeIds = toNodeIdMap(newShards); - - runCommit(transactionId, handle -> { - lockTable(handle, tableId); - - if (updateTime.isEmpty() && handle.attach(MetadataDao.class).isMaintenanceBlockedLocked(tableId)) { - throw new TrinoException(TRANSACTION_CONFLICT, "Maintenance is blocked for table"); - } - - ShardStats newStats = shardStats(newShards); - long rowCount = newStats.getRowCount(); - long compressedSize = newStats.getCompressedSize(); - long uncompressedSize = newStats.getUncompressedSize(); - - for (List shards : partition(newShards, 1000)) { - insertShardsAndIndex(tableId, columns, shards, nodeIds, handle); - } - - for (List uuids : partition(oldShardUuids, 1000)) { - ShardStats stats = deleteShardsAndIndex(tableId, ImmutableSet.copyOf(uuids), handle); - rowCount -= stats.getRowCount(); - compressedSize -= stats.getCompressedSize(); - uncompressedSize -= stats.getUncompressedSize(); - } - - long shardCount = newShards.size() - oldShardUuids.size(); - - if (!oldShardUuids.isEmpty() || !newShards.isEmpty()) { - MetadataDao metadata = handle.attach(MetadataDao.class); - metadata.updateTableStats(tableId, shardCount, rowCount, compressedSize, uncompressedSize); - updateTime.ifPresent(time -> metadata.updateTableVersion(tableId, time)); - } - }); - } - - private void runCommit(long transactionId, HandleConsumer callback) - { - int maxAttempts = 5; - for (int attempt = 1; attempt <= maxAttempts; attempt++) { - try { - dbi.useTransaction(handle -> { - ShardDao dao = shardDaoSupplier.attach(handle); - if (commitTransaction(dao, transactionId)) { - callback.useHandle(handle); - dao.deleteCreatedShards(transactionId); - } - }); - return; - } - catch (JdbiException | SQLException e) { - if (isTransactionCacheFullError(e)) { - throw metadataError(e, "Transaction too large"); - } - - if (e.getCause() != null) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - } - - if (attempt == maxAttempts) { - throw metadataError(e); - } - log.warn(e, "Failed to commit shards on attempt %d, will retry.", attempt); - try { - SECONDS.sleep(multiplyExact(attempt, 2)); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw metadataError(ie); - } - } - } - } - - private static boolean commitTransaction(ShardDao dao, long transactionId) - { - if (dao.finalizeTransaction(transactionId, true) != 1) { - if (TRUE.equals(dao.transactionSuccessful(transactionId))) { - return false; - } - throw new TrinoException(TRANSACTION_CONFLICT, "Transaction commit failed. Please retry the operation."); - } - return true; - } - - private ShardStats deleteShardsAndIndex(long tableId, Set shardUuids, Handle handle) - throws SQLException - { - String args = Joiner.on(",").join(nCopies(shardUuids.size(), "?")); - - ImmutableSet.Builder shardIdSet = ImmutableSet.builder(); - long rowCount = 0; - long compressedSize = 0; - long uncompressedSize = 0; - - String selectShards = format("" + - "SELECT shard_id, row_count, compressed_size, uncompressed_size\n" + - "FROM shards\n" + - "WHERE shard_uuid IN (%s)", args); - - try (PreparedStatement statement = handle.getConnection().prepareStatement(selectShards)) { - bindUuids(statement, shardUuids); - try (ResultSet rs = statement.executeQuery()) { - while (rs.next()) { - shardIdSet.add(rs.getLong("shard_id")); - rowCount += rs.getLong("row_count"); - compressedSize += rs.getLong("compressed_size"); - uncompressedSize += rs.getLong("uncompressed_size"); - } - } - } - - Set shardIds = shardIdSet.build(); - if (shardIds.size() != shardUuids.size()) { - throw transactionConflict(); - } - - ShardDao dao = shardDaoSupplier.attach(handle); - dao.insertDeletedShards(shardUuids); - - String where = " WHERE shard_id IN (" + args + ")"; - String deleteFromShardNodes = "DELETE FROM shard_nodes " + where; - String deleteFromShards = "DELETE FROM shards " + where; - String deleteFromShardIndex = "DELETE FROM " + shardIndexTable(tableId) + where; - - try (PreparedStatement statement = handle.getConnection().prepareStatement(deleteFromShardNodes)) { - bindLongs(statement, shardIds); - statement.executeUpdate(); - } - - for (String sql : asList(deleteFromShards, deleteFromShardIndex)) { - try (PreparedStatement statement = handle.getConnection().prepareStatement(sql)) { - bindLongs(statement, shardIds); - if (statement.executeUpdate() != shardIds.size()) { - throw transactionConflict(); - } - } - } - - return new ShardStats(rowCount, compressedSize, uncompressedSize); - } - - private static void bindUuids(PreparedStatement statement, Iterable uuids) - throws SQLException - { - int i = 1; - for (UUID uuid : uuids) { - statement.setBytes(i, uuidToBytes(uuid)); - i++; - } - } - - private static void bindLongs(PreparedStatement statement, Iterable values) - throws SQLException - { - int i = 1; - for (long value : values) { - statement.setLong(i, value); - i++; - } - } - - private static void insertShardsAndIndex(long tableId, List columns, Collection shards, Map nodeIds, Handle handle) - throws SQLException - { - if (shards.isEmpty()) { - return; - } - boolean bucketed = shards.iterator().next().getBucketNumber().isPresent(); - - Connection connection = handle.getConnection(); - try (IndexInserter indexInserter = new IndexInserter(connection, tableId, columns)) { - for (List batch : partition(shards, batchSize(connection))) { - List shardIds = insertShards(connection, tableId, batch); - - if (!bucketed) { - insertShardNodes(connection, nodeIds, shardIds, batch); - } - - for (int i = 0; i < batch.size(); i++) { - ShardInfo shard = batch.get(i); - Set shardNodes = shard.getNodeIdentifiers().stream() - .map(nodeIds::get) - .collect(toSet()); - indexInserter.insert( - shardIds.get(i), - shard.getShardUuid(), - shard.getBucketNumber(), - shardNodes, - shard.getColumnStats()); - } - indexInserter.execute(); - } - } - } - - private static int batchSize(Connection connection) - { - // H2 does not return generated keys properly - // https://github.com/h2database/h2database/issues/156 - return (connection instanceof JdbcConnection) ? 1 : 1000; - } - - private Map toNodeIdMap(Collection shards) - { - Set identifiers = shards.stream() - .map(ShardInfo::getNodeIdentifiers) - .flatMap(Collection::stream) - .collect(toSet()); - return Maps.toMap(identifiers, this::getOrCreateNodeId); - } - - @Override - public ShardMetadata getShard(UUID shardUuid) - { - return dao.getShard(shardUuid); - } - - @Override - public Set getNodeShards(String nodeIdentifier) - { - return dao.getNodeShards(nodeIdentifier, null); - } - - @Override - public Set getNodeShards(String nodeIdentifier, long tableId) - { - return dao.getNodeShards(nodeIdentifier, tableId); - } - - @Override - public ResultIterator getShardNodes(long tableId, TupleDomain effectivePredicate) - { - return new ShardIterator(tableId, false, Optional.empty(), effectivePredicate, dbi); - } - - @Override - public ResultIterator getShardNodesBucketed(long tableId, boolean merged, List bucketToNode, TupleDomain effectivePredicate) - { - return new ShardIterator(tableId, merged, Optional.of(bucketToNode), effectivePredicate, dbi); - } - - @Override - public void replaceShardAssignment(long tableId, UUID shardUuid, String nodeIdentifier, boolean gracePeriod) - { - if (gracePeriod && (nanosSince(startTime).compareTo(startupGracePeriod) < 0)) { - throw new TrinoException(SERVER_STARTING_UP, "Cannot reassign shards while server is starting"); - } - - int nodeId = getOrCreateNodeId(nodeIdentifier); - - runTransaction(dbi, handle -> { - ShardDao dao = shardDaoSupplier.attach(handle); - - Set oldAssignments = new HashSet<>(fetchLockedNodeIds(handle, tableId, shardUuid)); - updateNodeIds(handle, tableId, shardUuid, ImmutableSet.of(nodeId)); - - dao.deleteShardNodes(shardUuid, oldAssignments); - dao.insertShardNode(shardUuid, nodeId); - return null; - }); - } - - @Override - public Map getNodeBytes() - { - return dao.getNodeSizes().stream() - .collect(toMap(NodeSize::getNodeIdentifier, NodeSize::getSizeInBytes)); - } - - @Override - public long beginTransaction() - { - return dao.insertTransaction(); - } - - @Override - public void rollbackTransaction(long transactionId) - { - dao.finalizeTransaction(transactionId, false); - } - - @Override - public void createBuckets(long distributionId, int bucketCount) - { - Iterator nodeIterator = cyclingShuffledIterator(getNodeIdentifiers()); - - List bucketNumbers = new ArrayList<>(); - List nodeIds = new ArrayList<>(); - for (int bucket = 0; bucket < bucketCount; bucket++) { - bucketNumbers.add(bucket); - nodeIds.add(getOrCreateNodeId(nodeIterator.next())); - } - - runIgnoringConstraintViolation(() -> dao.insertBuckets(distributionId, bucketNumbers, nodeIds)); - } - - @Override - public List getBucketAssignments(long distributionId) - { - try { - return bucketAssignmentsCache.getUnchecked(distributionId); - } - catch (UncheckedExecutionException e) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - throw e; - } - } - - @Override - public void updateBucketAssignment(long distributionId, int bucketNumber, String nodeId) - { - dao.updateBucketNode(distributionId, bucketNumber, getOrCreateNodeId(nodeId)); - } - - @Override - public List getDistributions() - { - return dao.listActiveDistributions(); - } - - @Override - public long getDistributionSizeInBytes(long distributionId) - { - return dao.getDistributionSizeBytes(distributionId); - } - - @Override - public List getBucketNodes(long distibutionId) - { - return dao.getBucketNodes(distibutionId); - } - - @Override - public Set getExistingShardUuids(long tableId, Set shardUuids) - { - try (Handle handle = dbi.open()) { - String args = Joiner.on(",").join(nCopies(shardUuids.size(), "?")); - String selectShards = format( - "SELECT shard_uuid FROM %s WHERE shard_uuid IN (%s)", - shardIndexTable(tableId), args); - - ImmutableSet.Builder existingShards = ImmutableSet.builder(); - try (PreparedStatement statement = handle.getConnection().prepareStatement(selectShards)) { - bindUuids(statement, shardUuids); - try (ResultSet rs = statement.executeQuery()) { - while (rs.next()) { - existingShards.add(uuidFromBytes(rs.getBytes("shard_uuid"))); - } - } - } - return existingShards.build(); - } - catch (SQLException e) { - throw new RuntimeException(e); - } - } - - private List getBuckets(long distributionId) - { - return dao.getBucketNodes(distributionId); - } - - private List loadBucketAssignments(long distributionId) - { - Set nodeIds = getNodeIdentifiers(); - List bucketNodes = getBuckets(distributionId); - BucketReassigner reassigner = new BucketReassigner(nodeIds, bucketNodes); - - List assignments = new ArrayList<>(nCopies(bucketNodes.size(), null)); - TrinoException limiterException = null; - Set offlineNodes = new HashSet<>(); - - for (BucketNode bucketNode : bucketNodes) { - int bucket = bucketNode.getBucketNumber(); - String nodeId = bucketNode.getNodeIdentifier(); - - if (!nodeIds.contains(nodeId)) { - if (nanosSince(startTime).compareTo(startupGracePeriod) < 0) { - throw new TrinoException(SERVER_STARTING_UP, "Cannot reassign buckets while server is starting"); - } - - try { - if (offlineNodes.add(nodeId)) { - assignmentLimiter.checkAssignFrom(nodeId); - } - } - catch (TrinoException e) { - if (limiterException == null) { - limiterException = e; - } - continue; - } - - String oldNodeId = nodeId; - nodeId = reassigner.getNextReassignmentDestination(); - dao.updateBucketNode(distributionId, bucket, getOrCreateNodeId(nodeId)); - log.info("Reassigned bucket %s for distribution ID %s from %s to %s", bucket, distributionId, oldNodeId, nodeId); - } - - verify(assignments.set(bucket, nodeId) == null, "Duplicate bucket"); - } - - if (limiterException != null) { - throw limiterException; - } - - return ImmutableList.copyOf(assignments); - } - - private Set getNodeIdentifiers() - { - Set nodeIds = nodeSupplier.getWorkerNodes().stream() - .map(Node::getNodeIdentifier) - .collect(toSet()); - if (nodeIds.isEmpty()) { - throw new TrinoException(NO_NODES_AVAILABLE, "No nodes available for bucket assignments"); - } - return nodeIds; - } - - private int getOrCreateNodeId(String nodeIdentifier) - { - try { - return nodeIdCache.getUnchecked(nodeIdentifier); - } - catch (UncheckedExecutionException e) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - throw e; - } - } - - private int loadNodeId(String nodeIdentifier) - { - Integer id = dao.getNodeId(nodeIdentifier); - if (id != null) { - return id; - } - - // creating a node is idempotent - runIgnoringConstraintViolation(() -> dao.insertNode(nodeIdentifier)); - - id = dao.getNodeId(nodeIdentifier); - if (id == null) { - throw new TrinoException(GENERIC_INTERNAL_ERROR, "node does not exist after insert"); - } - return id; - } - - private Duration nanosSince(long nanos) - { - return new Duration(ticker.read() - nanos, NANOSECONDS); - } - - private static List insertShards(Connection connection, long tableId, List shards) - throws SQLException - { - String sql = "" + - "INSERT INTO shards (shard_uuid, table_id, create_time, row_count, compressed_size, uncompressed_size, xxhash64, bucket_number)\n" + - "VALUES (?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?)"; - - try (PreparedStatement statement = connection.prepareStatement(sql, RETURN_GENERATED_KEYS)) { - for (ShardInfo shard : shards) { - statement.setBytes(1, uuidToBytes(shard.getShardUuid())); - statement.setLong(2, tableId); - statement.setLong(3, shard.getRowCount()); - statement.setLong(4, shard.getCompressedSize()); - statement.setLong(5, shard.getUncompressedSize()); - statement.setLong(6, shard.getXxhash64()); - bindOptionalInt(statement, 7, shard.getBucketNumber()); - statement.addBatch(); - } - statement.executeBatch(); - - ImmutableList.Builder builder = ImmutableList.builder(); - try (ResultSet keys = statement.getGeneratedKeys()) { - while (keys.next()) { - builder.add(keys.getLong(1)); - } - } - List shardIds = builder.build(); - - if (shardIds.size() != shards.size()) { - throw new TrinoException(RAPTOR_ERROR, "Wrong number of generated keys for inserted shards"); - } - return shardIds; - } - } - - private static void insertShardNodes(Connection connection, Map nodeIds, List shardIds, List shards) - throws SQLException - { - checkArgument(shardIds.size() == shards.size(), "lists are not the same size"); - String sql = "INSERT INTO shard_nodes (shard_id, node_id) VALUES (?, ?)"; - try (PreparedStatement statement = connection.prepareStatement(sql)) { - for (int i = 0; i < shards.size(); i++) { - for (String identifier : shards.get(i).getNodeIdentifiers()) { - statement.setLong(1, shardIds.get(i)); - statement.setInt(2, nodeIds.get(identifier)); - statement.addBatch(); - } - } - statement.executeBatch(); - } - } - - private static Collection fetchLockedNodeIds(Handle handle, long tableId, UUID shardUuid) - { - String sql = format( - "SELECT node_ids FROM %s WHERE shard_uuid = ? FOR UPDATE", - shardIndexTable(tableId)); - - byte[] nodeArray = handle.createQuery(sql) - .bind(0, uuidToBytes(shardUuid)) - .mapTo(byte[].class) - .one(); - - return intArrayFromBytes(nodeArray); - } - - private static void updateNodeIds(Handle handle, long tableId, UUID shardUuid, Set nodeIds) - { - String sql = format( - "UPDATE %s SET node_ids = ? WHERE shard_uuid = ?", - shardIndexTable(tableId)); - - handle.execute(sql, intArrayToBytes(nodeIds), uuidToBytes(shardUuid)); - } - - private static void lockTable(Handle handle, long tableId) - { - if (handle.attach(MetadataDao.class).getLockedTableId(tableId) == null) { - throw transactionConflict(); - } - } - - private static TrinoException transactionConflict() - { - return new TrinoException(TRANSACTION_CONFLICT, "Table was updated by a different transaction. Please retry the operation."); - } - - public static String shardIndexTable(long tableId) - { - return INDEX_TABLE_PREFIX + tableId; - } - - public static String minColumn(long columnId) - { - checkArgument(columnId >= 0, "invalid columnId %s", columnId); - return format("c%s_min", columnId); - } - - public static String maxColumn(long columnId) - { - checkArgument(columnId >= 0, "invalid columnId %s", columnId); - return format("c%s_max", columnId); - } - - private static String sqlColumnType(Type type) - { - JDBCType jdbcType = jdbcType(type); - if (jdbcType != null) { - switch (jdbcType) { - case BOOLEAN: - return "boolean"; - case BIGINT: - return "bigint"; - case DOUBLE: - return "double"; - case INTEGER: - return "int"; - case VARBINARY: - return format("varbinary(%s)", MAX_BINARY_INDEX_SIZE); - default: - break; - } - } - return null; - } - - private static Iterator cyclingShuffledIterator(Collection collection) - { - List list = new ArrayList<>(collection); - Collections.shuffle(list); - return Iterables.cycle(list).iterator(); - } - - private static ShardStats shardStats(Collection shards) - { - return new ShardStats( - shards.stream().mapToLong(ShardInfo::getRowCount).sum(), - shards.stream().mapToLong(ShardInfo::getCompressedSize).sum(), - shards.stream().mapToLong(ShardInfo::getUncompressedSize).sum()); - } - - private static class ShardStats - { - private final long rowCount; - private final long compressedSize; - private final long uncompressedSize; - - public ShardStats(long rowCount, long compressedSize, long uncompressedSize) - { - this.rowCount = rowCount; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - } - - public long getRowCount() - { - return rowCount; - } - - public long getCompressedSize() - { - return compressedSize; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardRecorder.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardRecorder.java deleted file mode 100644 index 801a570e6c82..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardRecorder.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.TrinoException; - -import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; - -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runIgnoringConstraintViolation; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class DatabaseShardRecorder - implements ShardRecorder -{ - private static final Logger log = Logger.get(DatabaseShardRecorder.class); - - private final ShardDao dao; - - @Inject - public DatabaseShardRecorder(DaoSupplier shardDaoSupplier) - { - this.dao = shardDaoSupplier.onDemand(); - } - - @Override - public void recordCreatedShard(long transactionId, UUID shardUuid) - { - int maxAttempts = 5; - for (int attempt = 1; attempt <= maxAttempts; attempt++) { - try { - runIgnoringConstraintViolation(() -> dao.insertCreatedShard(shardUuid, transactionId)); - return; - } - catch (TrinoException e) { - if (attempt == maxAttempts) { - throw e; - } - log.warn(e, "Failed to insert created shard on attempt %s, will retry", attempt); - try { - long millis = attempt * 2000L; - MILLISECONDS.sleep(millis + ThreadLocalRandom.current().nextLong(0, millis)); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw metadataError(ie); - } - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Distribution.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Distribution.java deleted file mode 100644 index 9ee87bb9f269..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Distribution.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.json.JsonCodec; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeManager; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; -import java.util.Optional; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.json.JsonCodec.listJsonCodec; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class Distribution -{ - private static final JsonCodec> LIST_CODEC = listJsonCodec(String.class); - - private final long id; - private final Optional name; - private final List columnTypes; - private final int bucketCount; - - public Distribution(long id, Optional name, List columnTypes, int bucketCount) - { - this.id = id; - this.name = requireNonNull(name, "name is null"); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.bucketCount = bucketCount; - } - - public long getId() - { - return id; - } - - public Optional getName() - { - return name; - } - - public List getColumnTypes() - { - return columnTypes; - } - - public int getBucketCount() - { - return bucketCount; - } - - public static class Mapper - implements RowMapper - { - private final TypeManager typeManager; - - @Inject - public Mapper(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public Distribution map(ResultSet rs, StatementContext ctx) - throws SQLException - { - List types = LIST_CODEC.fromJson(rs.getString("column_types")).stream() - .map(TypeId::of) - .map(typeManager::getType) - .collect(toImmutableList()); - - return new Distribution( - rs.getLong("distribution_id"), - Optional.ofNullable(rs.getString("distribution_name")), - types, - rs.getInt("bucket_count")); - } - } - - public static String serializeColumnTypes(List columnTypes) - { - return LIST_CODEC.toJson(columnTypes.stream() - .map(type -> type.getTypeId().getId()) - .collect(toList())); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ForMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ForMetadata.java deleted file mode 100644 index ee2b31918ddb..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ForMetadata.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.inject.BindingAnnotation; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -@Retention(RUNTIME) -@Target({FIELD, PARAMETER, METHOD}) -@BindingAnnotation -public @interface ForMetadata -{ -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2DatabaseConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2DatabaseConfig.java deleted file mode 100644 index e00ef7eea310..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2DatabaseConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import jakarta.validation.constraints.NotNull; - -public class H2DatabaseConfig -{ - private String filename; - - @NotNull - public String getFilename() - { - return filename; - } - - @Config("metadata.db.filename") - public H2DatabaseConfig setFilename(String filename) - { - this.filename = filename; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2ShardDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2ShardDao.java deleted file mode 100644 index 1dc04f70737d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2ShardDao.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.sqlobject.customizer.Bind; -import org.jdbi.v3.sqlobject.statement.SqlBatch; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.sql.Timestamp; -import java.util.UUID; - -public interface H2ShardDao - extends ShardDao -{ - @Override - @SqlBatch("MERGE INTO deleted_shards (shard_uuid, delete_time)\n" + - "VALUES (:shardUuid, CURRENT_TIMESTAMP)") - void insertDeletedShards(@Bind("shardUuid") Iterable shardUuids); - - @Override - @SqlUpdate("DELETE FROM transactions\n" + - "WHERE end_time < :maxEndTime\n" + - " AND successful IN (TRUE, FALSE)\n" + - " AND transaction_id NOT IN (SELECT transaction_id FROM created_shards)\n" + - "LIMIT " + CLEANUP_TRANSACTIONS_BATCH_SIZE) - int deleteOldCompletedTransactions(Timestamp maxEndTime); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/IndexInserter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/IndexInserter.java deleted file mode 100644 index aeadbdaade5a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/IndexInserter.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.jdbi.v3.core.Jdbi; - -import java.sql.Connection; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.OptionalInt; -import java.util.Set; -import java.util.StringJoiner; -import java.util.UUID; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isHiddenColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.metadata.ShardPredicate.bindValue; -import static io.trino.plugin.raptor.legacy.storage.ColumnIndexStatsUtils.jdbcType; -import static io.trino.plugin.raptor.legacy.util.ArrayUtil.intArrayToBytes; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidToBytes; - -class IndexInserter - implements AutoCloseable -{ - private final boolean bucketed; - private final List columns; - private final Map indexes; - private final Map types; - private final PreparedStatement statement; - - public IndexInserter(Connection connection, long tableId, List columns) - throws SQLException - { - this.bucketed = Jdbi.open(connection) - .select("SELECT distribution_id IS NOT NULL FROM tables WHERE table_id = ?", tableId) - .mapTo(boolean.class) - .one(); - - ImmutableList.Builder columnBuilder = ImmutableList.builder(); - ImmutableMap.Builder indexBuilder = ImmutableMap.builder(); - ImmutableMap.Builder typeBuilder = ImmutableMap.builder(); - StringJoiner nameJoiner = new StringJoiner(", "); - StringJoiner valueJoiner = new StringJoiner(", "); - int index = 1; - - nameJoiner.add("shard_id").add("shard_uuid"); - valueJoiner.add("?").add("?").add("?"); - index += 3; - - if (bucketed) { - nameJoiner.add("bucket_number"); - } - else { - nameJoiner.add("node_ids"); - } - - for (ColumnInfo column : columns) { - JDBCType jdbcType = jdbcType(column.getType()); - if (jdbcType == null) { - continue; - } - - long columnId = column.getColumnId(); - if (isHiddenColumn(columnId)) { - continue; - } - - columnBuilder.add(column); - nameJoiner.add(minColumn(columnId)); - nameJoiner.add(maxColumn(columnId)); - valueJoiner.add("?").add("?"); - - indexBuilder.put(columnId, index); - index += 2; - - typeBuilder.put(columnId, jdbcType); - } - - this.columns = columnBuilder.build(); - this.indexes = indexBuilder.buildOrThrow(); - this.types = typeBuilder.buildOrThrow(); - - String sql = "" + - "INSERT INTO " + shardIndexTable(tableId) + "\n" + - "(" + nameJoiner + ")\n" + - "VALUES (" + valueJoiner + ")"; - - this.statement = connection.prepareStatement(sql); - } - - @Override - public void close() - throws SQLException - { - statement.close(); - } - - public void insert(long shardId, UUID shardUuid, OptionalInt bucketNumber, Set nodeIds, List stats) - throws SQLException - { - statement.setLong(1, shardId); - statement.setBytes(2, uuidToBytes(shardUuid)); - - if (bucketed) { - checkArgument(bucketNumber.isPresent(), "shard bucket missing for bucketed table"); - statement.setInt(3, bucketNumber.getAsInt()); - } - else { - checkArgument(bucketNumber.isEmpty(), "shard bucket present for non-bucketed table"); - statement.setBytes(3, intArrayToBytes(nodeIds)); - } - - for (ColumnInfo column : columns) { - int index = indexes.get(column.getColumnId()); - int type = types.get(column.getColumnId()).getVendorTypeNumber(); - statement.setNull(index, type); - statement.setNull(index + 1, type); - } - - for (ColumnStats column : stats) { - if (!indexes.containsKey(column.getColumnId())) { - // the column no longer exists in the table - continue; - } - int index = indexes.get(column.getColumnId()); - JDBCType type = types.get(column.getColumnId()); - bindValue(statement, type, convert(column.getMin()), index); - bindValue(statement, type, convert(column.getMax()), index + 1); - } - - statement.addBatch(); - } - - public void execute() - throws SQLException - { - statement.executeBatch(); - } - - private static Object convert(Object value) - { - if (value instanceof String) { - return utf8Slice((String) value); - } - return value; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/JdbcDatabaseConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/JdbcDatabaseConfig.java deleted file mode 100644 index 0ff540f8631b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/JdbcDatabaseConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import jakarta.validation.constraints.NotNull; - -public class JdbcDatabaseConfig -{ - private String url; - - @NotNull - public String getUrl() - { - return url; - } - - @Config("metadata.db.url") - public JdbcDatabaseConfig setUrl(String url) - { - this.url = url; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataConfig.java deleted file mode 100644 index 55a1dca3e251..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataConfig.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; -import jakarta.validation.constraints.NotNull; - -import static java.util.concurrent.TimeUnit.MINUTES; - -public class MetadataConfig -{ - private Duration startupGracePeriod = new Duration(5, MINUTES); - private Duration reassignmentDelay = new Duration(0, MINUTES); - private Duration reassignmentInterval = new Duration(0, MINUTES); - - @NotNull - public Duration getStartupGracePeriod() - { - return startupGracePeriod; - } - - @Config("raptor.startup-grace-period") - @ConfigDescription("Minimum uptime before allowing bucket or shard reassignments") - public MetadataConfig setStartupGracePeriod(Duration startupGracePeriod) - { - this.startupGracePeriod = startupGracePeriod; - return this; - } - - @NotNull - public Duration getReassignmentDelay() - { - return reassignmentDelay; - } - - @Config("raptor.reassignment-delay") - @ConfigDescription("Minimum delay before allowing reassignments for a node") - public MetadataConfig setReassignmentDelay(Duration reassignmentDelay) - { - this.reassignmentDelay = reassignmentDelay; - return this; - } - - @NotNull - public Duration getReassignmentInterval() - { - return reassignmentInterval; - } - - @Config("raptor.reassignment-interval") - @ConfigDescription("Minimum interval between reassignments for different nodes") - public MetadataConfig setReassignmentInterval(Duration reassignmentInterval) - { - this.reassignmentInterval = reassignmentInterval; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataDao.java deleted file mode 100644 index 49c3cb7ad5c9..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataDao.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.plugin.raptor.legacy.metadata.Table.TableMapper; -import io.trino.spi.connector.SchemaTableName; -import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; -import org.jdbi.v3.sqlobject.config.RegisterRowMapper; -import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.util.List; -import java.util.Set; - -@RegisterConstructorMapper(ColumnMetadataRow.class) -@RegisterConstructorMapper(TableMetadataRow.class) -@RegisterConstructorMapper(TableStatsRow.class) -@RegisterRowMapper(SchemaTableNameMapper.class) -@RegisterRowMapper(TableMapper.class) -@RegisterRowMapper(ViewResult.Mapper.class) -public interface MetadataDao -{ - String TABLE_INFORMATION_SELECT = "" + - "SELECT t.table_id, t.distribution_id, d.distribution_name, d.bucket_count, t.temporal_column_id, t.organization_enabled\n" + - "FROM tables t\n" + - "LEFT JOIN distributions d ON (t.distribution_id = d.distribution_id)\n"; - - String TABLE_COLUMN_SELECT = "" + - "SELECT t.schema_name, t.table_name,\n" + - " c.column_id, c.column_name, c.data_type, c.ordinal_position,\n" + - " c.bucket_ordinal_position, c.sort_ordinal_position,\n" + - " t.temporal_column_id = c.column_id AS temporal\n" + - "FROM tables t\n" + - "JOIN columns c ON (t.table_id = c.table_id)\n"; - - @SqlQuery(TABLE_INFORMATION_SELECT + - "WHERE t.table_id = :tableId") - Table getTableInformation(long tableId); - - @SqlQuery(TABLE_INFORMATION_SELECT + - "WHERE t.schema_name = :schemaName\n" + - " AND t.table_name = :tableName") - Table getTableInformation( - String schemaName, - String tableName); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE t.table_id = :tableId\n" + - " AND c.column_id = :columnId\n" + - "ORDER BY c.ordinal_position\n") - TableColumn getTableColumn( - long tableId, - long columnId); - - @SqlQuery("SELECT schema_name, table_name\n" + - "FROM tables\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)") - List listTables( - String schemaName); - - @SqlQuery("SELECT DISTINCT schema_name FROM tables") - List listSchemaNames(); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL)\n" + - "ORDER BY schema_name, table_name, ordinal_position") - List listTableColumns( - String schemaName, - String tableName); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE t.table_id = :tableId\n" + - "ORDER BY c.ordinal_position") - List listTableColumns(long tableId); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE t.table_id = :tableId\n" + - " AND c.sort_ordinal_position IS NOT NULL\n" + - "ORDER BY c.sort_ordinal_position") - List listSortColumns(long tableId); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE t.table_id = :tableId\n" + - " AND c.bucket_ordinal_position IS NOT NULL\n" + - "ORDER BY c.bucket_ordinal_position") - List listBucketColumns(long tableId); - - @SqlQuery("SELECT schema_name, table_name, data\n" + - "FROM views\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)") - List listViews( - String schemaName); - - @SqlQuery("SELECT schema_name, table_name, data\n" + - "FROM views\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL)\n" + - "ORDER BY schema_name, table_name\n") - List getViews( - String schemaName, - String tableName); - - @SqlUpdate("INSERT INTO tables (\n" + - " schema_name, table_name, compaction_enabled, organization_enabled, distribution_id,\n" + - " create_time, update_time, table_version,\n" + - " shard_count, row_count, compressed_size, uncompressed_size)\n" + - "VALUES (\n" + - " :schemaName, :tableName, :compactionEnabled, :organizationEnabled, :distributionId,\n" + - " :createTime, :createTime, 0,\n" + - " 0, 0, 0, 0)\n") - @GetGeneratedKeys - long insertTable( - String schemaName, - String tableName, - boolean compactionEnabled, - boolean organizationEnabled, - Long distributionId, - long createTime); - - @SqlUpdate("UPDATE tables SET\n" + - " update_time = :updateTime\n" + - ", table_version = table_version + 1\n" + - "WHERE table_id = :tableId") - void updateTableVersion( - long tableId, - long updateTime); - - @SqlUpdate("UPDATE tables SET\n" + - " shard_count = shard_count + :shardCount \n" + - ", row_count = row_count + :rowCount\n" + - ", compressed_size = compressed_size + :compressedSize\n" + - ", uncompressed_size = uncompressed_size + :uncompressedSize\n" + - "WHERE table_id = :tableId") - void updateTableStats( - long tableId, - long shardCount, - long rowCount, - long compressedSize, - long uncompressedSize); - - @SqlUpdate("INSERT INTO columns (table_id, column_id, column_name, ordinal_position, data_type, sort_ordinal_position, bucket_ordinal_position)\n" + - "VALUES (:tableId, :columnId, :columnName, :ordinalPosition, :dataType, :sortOrdinalPosition, :bucketOrdinalPosition)") - void insertColumn( - long tableId, - long columnId, - String columnName, - int ordinalPosition, - String dataType, - Integer sortOrdinalPosition, - Integer bucketOrdinalPosition); - - @SqlUpdate("UPDATE tables SET\n" + - " schema_name = :newSchemaName\n" + - ", table_name = :newTableName\n" + - "WHERE table_id = :tableId") - void renameTable( - long tableId, - String newSchemaName, - String newTableName); - - @SqlUpdate("UPDATE columns SET column_name = :target\n" + - "WHERE table_id = :tableId\n" + - " AND column_id = :columnId") - void renameColumn( - long tableId, - long columnId, - String target); - - @SqlUpdate("DELETE FROM columns\n" + - " WHERE table_id = :tableId\n" + - " AND column_id = :columnId") - void dropColumn( - long tableId, - long columnId); - - @SqlUpdate("INSERT INTO views (schema_name, table_name, data)\n" + - "VALUES (:schemaName, :tableName, :data)") - void insertView( - String schemaName, - String tableName, - String data); - - @SqlUpdate("DELETE FROM tables WHERE table_id = :tableId") - int dropTable(long tableId); - - @SqlUpdate("DELETE FROM columns WHERE table_id = :tableId") - int dropColumns(long tableId); - - @SqlUpdate("DELETE FROM views\n" + - "WHERE schema_name = :schemaName\n" + - " AND table_name = :tableName") - int dropView( - String schemaName, - String tableName); - - @SqlQuery("SELECT temporal_column_id\n" + - "FROM tables\n" + - "WHERE table_id = :tableId") - Long getTemporalColumnId(long tableId); - - @SqlUpdate("UPDATE tables SET\n" + - "temporal_column_id = :columnId\n" + - "WHERE table_id = :tableId") - void updateTemporalColumnId( - long tableId, - long columnId); - - @SqlQuery("SELECT compaction_enabled AND maintenance_blocked IS NULL\n" + - "FROM tables\n" + - "WHERE table_id = :tableId") - boolean isCompactionEligible(long tableId); - - @SqlQuery("SELECT table_id FROM tables WHERE table_id = :tableId FOR UPDATE") - Long getLockedTableId(long tableId); - - @SqlQuery("SELECT distribution_id, distribution_name, column_types, bucket_count\n" + - "FROM distributions\n" + - "WHERE distribution_id = :distributionId") - Distribution getDistribution(long distributionId); - - @SqlQuery("SELECT distribution_id, distribution_name, column_types, bucket_count\n" + - "FROM distributions\n" + - "WHERE distribution_name = :distributionName") - Distribution getDistribution(String distributionName); - - @SqlUpdate("INSERT INTO distributions (distribution_name, column_types, bucket_count)\n" + - "VALUES (:distributionName, :columnTypes, :bucketCount)") - @GetGeneratedKeys - long insertDistribution( - String distributionName, - String columnTypes, - int bucketCount); - - @SqlQuery("SELECT table_id, schema_name, table_name, temporal_column_id, distribution_name, bucket_count, organization_enabled\n" + - "FROM tables\n" + - "LEFT JOIN distributions\n" + - "ON tables.distribution_id = distributions.distribution_id\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL)\n" + - "ORDER BY table_id") - List getTableMetadataRows( - String schemaName, - String tableName); - - @SqlQuery("SELECT table_id, column_id, column_name, sort_ordinal_position, bucket_ordinal_position\n" + - "FROM columns\n" + - "WHERE table_id IN (\n" + - " SELECT table_id\n" + - " FROM tables\n" + - " WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL))\n" + - "ORDER BY table_id") - List getColumnMetadataRows( - String schemaName, - String tableName); - - @SqlQuery("SELECT schema_name, table_name, create_time, update_time, table_version,\n" + - " shard_count, row_count, compressed_size, uncompressed_size\n" + - "FROM tables\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL)\n" + - "ORDER BY schema_name, table_name") - List getTableStatsRows( - String schemaName, - String tableName); - - @SqlQuery("SELECT table_id\n" + - "FROM tables\n" + - "WHERE organization_enabled\n" + - " AND maintenance_blocked IS NULL\n" + - " AND table_id IN\n" + - " (SELECT table_id\n" + - " FROM columns\n" + - " WHERE sort_ordinal_position IS NOT NULL)") - Set getOrganizationEligibleTables(); - - @SqlUpdate("UPDATE tables SET maintenance_blocked = CURRENT_TIMESTAMP\n" + - "WHERE table_id = :tableId\n" + - " AND maintenance_blocked IS NULL") - void blockMaintenance(long tableId); - - @SqlUpdate("UPDATE tables SET maintenance_blocked = NULL\n" + - "WHERE table_id = :tableId") - void unblockMaintenance(long tableId); - - @SqlQuery("SELECT maintenance_blocked IS NOT NULL\n" + - "FROM tables\n" + - "WHERE table_id = :tableId\n" + - "FOR UPDATE") - boolean isMaintenanceBlockedLocked(long tableId); - - @SqlUpdate("UPDATE tables SET maintenance_blocked = NULL\n" + - "WHERE maintenance_blocked IS NOT NULL") - void unblockAllMaintenance(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MySqlShardDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MySqlShardDao.java deleted file mode 100644 index 0df2cee91ea8..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MySqlShardDao.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.sqlobject.customizer.Bind; -import org.jdbi.v3.sqlobject.statement.SqlBatch; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.sql.Timestamp; -import java.util.UUID; - -public interface MySqlShardDao - extends ShardDao -{ - @Override - @SqlUpdate("DELETE x\n" + - "FROM shard_nodes x\n" + - "JOIN shards USING (shard_id)\n" + - "WHERE table_id = :tableId") - void dropShardNodes(long tableId); - - @Override - @SqlBatch("INSERT IGNORE INTO deleted_shards (shard_uuid, delete_time)\n" + - "VALUES (:shardUuid, CURRENT_TIMESTAMP)") - void insertDeletedShards(@Bind("shardUuid") Iterable shardUuids); - - @Override - // 'order by' is needed in this statement in order to make it compatible with statement-based replication - @SqlUpdate("DELETE FROM transactions\n" + - "WHERE end_time < :maxEndTime\n" + - " AND successful IN (TRUE, FALSE)\n" + - " AND transaction_id NOT IN (SELECT transaction_id FROM created_shards)\n" + - "ORDER BY end_time, transaction_id\n" + - "LIMIT " + CLEANUP_TRANSACTIONS_BATCH_SIZE) - int deleteOldCompletedTransactions(Timestamp maxEndTime); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/NodeSize.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/NodeSize.java deleted file mode 100644 index 9f17495881bb..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/NodeSize.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.core.mapper.reflect.ColumnName; - -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class NodeSize -{ - private final String nodeIdentifier; - private final long sizeInBytes; - - public NodeSize(String nodeIdentifier, @ColumnName("bytes") long sizeInBytes) - { - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - checkArgument(sizeInBytes >= 0, "sizeInBytes must be >= 0"); - this.sizeInBytes = sizeInBytes; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } - - public long getSizeInBytes() - { - return sizeInBytes; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if ((o == null) || (getClass() != o.getClass())) { - return false; - } - NodeSize nodeSize = (NodeSize) o; - return (sizeInBytes == nodeSize.sizeInBytes) && - Objects.equals(nodeIdentifier, nodeSize.nodeIdentifier); - } - - @Override - public int hashCode() - { - return Objects.hash(nodeIdentifier, sizeInBytes); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("nodeIdentifier", nodeIdentifier) - .add("sizeInBytes", sizeInBytes) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/RaptorNode.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/RaptorNode.java deleted file mode 100644 index 162385a36c2f..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/RaptorNode.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import static java.util.Objects.requireNonNull; - -public class RaptorNode -{ - private final int nodeId; - private final String nodeIdentifier; - - public RaptorNode(int nodeId, String nodeIdentifier) - { - this.nodeId = nodeId; - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - } - - public int getNodeId() - { - return nodeId; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDao.java deleted file mode 100644 index 00a7b54d7b6e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDao.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -public interface SchemaDao -{ - @SqlUpdate("CREATE TABLE IF NOT EXISTS distributions (\n" + - " distribution_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + - " distribution_name VARCHAR(255),\n" + - " column_types TEXT NOT NULL,\n" + - " bucket_count INT NOT NULL,\n" + - " UNIQUE (distribution_name)\n" + - ")") - void createTableDistributions(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS tables (\n" + - " table_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + - " schema_name VARCHAR(255) NOT NULL,\n" + - " table_name VARCHAR(255) NOT NULL,\n" + - " temporal_column_id BIGINT,\n" + - " compaction_enabled BOOLEAN NOT NULL,\n" + - " organization_enabled BOOLEAN NOT NULL,\n" + - " distribution_id BIGINT,\n" + - " create_time BIGINT NOT NULL,\n" + - " update_time BIGINT NOT NULL,\n" + - " table_version BIGINT NOT NULL,\n" + - " shard_count BIGINT NOT NULL,\n" + - " row_count BIGINT NOT NULL,\n" + - " compressed_size BIGINT NOT NULL,\n" + - " uncompressed_size BIGINT NOT NULL,\n" + - " maintenance_blocked DATETIME,\n" + - " UNIQUE (schema_name, table_name),\n" + - " UNIQUE (distribution_id, table_id),\n" + - " UNIQUE (maintenance_blocked, table_id),\n" + - " FOREIGN KEY (distribution_id) REFERENCES distributions (distribution_id)\n" + - ")") - void createTableTables(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS columns (\n" + - " table_id BIGINT NOT NULL,\n" + - " column_id BIGINT NOT NULL,\n" + - " column_name VARCHAR(255) NOT NULL,\n" + - " ordinal_position INT NOT NULL,\n" + - " data_type VARCHAR(255) NOT NULL,\n" + - " sort_ordinal_position INT,\n" + - " bucket_ordinal_position INT,\n" + - " PRIMARY KEY (table_id, column_id),\n" + - " UNIQUE (table_id, column_name),\n" + - " UNIQUE (table_id, ordinal_position),\n" + - " UNIQUE (table_id, sort_ordinal_position),\n" + - " UNIQUE (table_id, bucket_ordinal_position),\n" + - " FOREIGN KEY (table_id) REFERENCES tables (table_id)\n" + - ")") - void createTableColumns(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS views (\n" + - " schema_name VARCHAR(255) NOT NULL,\n" + - " table_name VARCHAR(255) NOT NULL,\n" + - " data TEXT NOT NULL,\n" + - " PRIMARY KEY (schema_name, table_name)\n" + - ")") - void createTableViews(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS nodes (\n" + - " node_id INT PRIMARY KEY AUTO_INCREMENT,\n" + - " node_identifier VARCHAR(255) NOT NULL,\n" + - " UNIQUE (node_identifier)\n" + - ")") - void createTableNodes(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS shards (\n" + - " shard_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " table_id BIGINT NOT NULL,\n" + - " bucket_number INT,\n" + - " create_time DATETIME NOT NULL,\n" + - " row_count BIGINT NOT NULL,\n" + - " compressed_size BIGINT NOT NULL,\n" + - " uncompressed_size BIGINT NOT NULL,\n" + - " xxhash64 BIGINT NOT NULL,\n" + - " UNIQUE (shard_uuid),\n" + - // include a covering index organized by table_id - " UNIQUE (table_id, bucket_number, shard_id, shard_uuid, create_time, row_count, compressed_size, uncompressed_size, xxhash64),\n" + - " FOREIGN KEY (table_id) REFERENCES tables (table_id)\n" + - ")") - void createTableShards(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS shard_nodes (\n" + - " shard_id BIGINT NOT NULL,\n" + - " node_id INT NOT NULL,\n" + - " PRIMARY KEY (shard_id, node_id),\n" + - " UNIQUE (node_id, shard_id),\n" + - " FOREIGN KEY (shard_id) REFERENCES shards (shard_id),\n" + - " FOREIGN KEY (node_id) REFERENCES nodes (node_id)\n" + - ")") - void createTableShardNodes(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS external_batches (\n" + - " external_batch_id VARCHAR(255) PRIMARY KEY,\n" + - " successful BOOLEAN NOT NULL\n" + - ")") - void createTableExternalBatches(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS transactions (\n" + - " transaction_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + - " successful BOOLEAN,\n" + - " start_time DATETIME NOT NULL,\n" + - " end_time DATETIME,\n" + - " UNIQUE (successful, start_time, transaction_id),\n" + - " UNIQUE (end_time, transaction_id, successful)\n" + - ")") - void createTableTransactions(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS created_shards (\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " transaction_id BIGINT NOT NULL,\n" + - " PRIMARY KEY (shard_uuid),\n" + - " UNIQUE (transaction_id, shard_uuid),\n" + - " FOREIGN KEY (transaction_id) REFERENCES transactions (transaction_id)\n" + - ")") - void createTableCreatedShards(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS deleted_shards (\n" + - " shard_uuid BINARY(16) PRIMARY KEY,\n" + - " delete_time DATETIME NOT NULL,\n" + - " UNIQUE (delete_time, shard_uuid)\n" + - ")") - void createTableDeletedShards(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS buckets (\n" + - " distribution_id BIGINT NOT NULL,\n" + - " bucket_number INT NOT NULL,\n" + - " node_id INT NOT NULL,\n" + - " PRIMARY KEY (distribution_id, bucket_number),\n" + - " UNIQUE (node_id, distribution_id, bucket_number),\n" + - " FOREIGN KEY (distribution_id) REFERENCES distributions (distribution_id),\n" + - " FOREIGN KEY (node_id) REFERENCES nodes (node_id)\n" + - ")") - void createTableBuckets(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS shard_organizer_jobs (\n" + - " node_identifier VARCHAR(255) NOT NULL,\n" + - " table_id BIGINT NOT NULL,\n" + - " last_start_time BIGINT,\n" + - " PRIMARY KEY (node_identifier, table_id),\n" + - " UNIQUE (table_id, node_identifier),\n" + - " FOREIGN KEY (table_id) REFERENCES tables (table_id)\n" + - ")") - void createTableShardOrganizerJobs(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDaoUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDaoUtil.java deleted file mode 100644 index 5442248a1441..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDaoUtil.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import org.jdbi.v3.core.ConnectionException; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; - -import java.util.concurrent.TimeUnit; - -public final class SchemaDaoUtil -{ - private static final Logger log = Logger.get(SchemaDaoUtil.class); - - private SchemaDaoUtil() {} - - public static void createTablesWithRetry(Jdbi dbi) - { - Duration delay = new Duration(2, TimeUnit.SECONDS); - while (true) { - try (Handle handle = dbi.open()) { - createTables(handle.attach(SchemaDao.class)); - return; - } - catch (ConnectionException e) { - log.warn("Failed to connect to database. Will retry again in %s. Exception: %s", delay, e.getMessage()); - sleep(delay); - } - } - } - - private static void createTables(SchemaDao dao) - { - dao.createTableDistributions(); - dao.createTableTables(); - dao.createTableColumns(); - dao.createTableViews(); - dao.createTableNodes(); - dao.createTableShards(); - dao.createTableShardNodes(); - dao.createTableExternalBatches(); - dao.createTableTransactions(); - dao.createTableCreatedShards(); - dao.createTableDeletedShards(); - dao.createTableBuckets(); - dao.createTableShardOrganizerJobs(); - } - - private static void sleep(Duration duration) - { - try { - Thread.sleep(duration.toMillis()); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaTableNameMapper.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaTableNameMapper.java deleted file mode 100644 index 61451b0e28b3..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaTableNameMapper.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.spi.connector.SchemaTableName; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; - -public class SchemaTableNameMapper - implements RowMapper -{ - @Override - public SchemaTableName map(ResultSet r, StatementContext ctx) - throws SQLException - { - return new SchemaTableName( - r.getString("schema_name"), - r.getString("table_name")); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleaner.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleaner.java deleted file mode 100644 index 6fa8169e00cc..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleaner.java +++ /dev/null @@ -1,519 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Ticker; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.storage.StorageService; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.NodeManager; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.io.File; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.collect.Sets.difference; -import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.metadata.ShardDao.CLEANABLE_SHARDS_BATCH_SIZE; -import static io.trino.plugin.raptor.legacy.metadata.ShardDao.CLEANUP_TRANSACTIONS_BATCH_SIZE; -import static java.lang.Math.min; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.runAsync; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toSet; - -public class ShardCleaner -{ - private static final Logger log = Logger.get(ShardCleaner.class); - - private final ShardDao dao; - private final String currentNode; - private final boolean coordinator; - private final Ticker ticker; - private final StorageService storageService; - private final Optional backupStore; - private final Duration maxTransactionAge; - private final Duration transactionCleanerInterval; - private final Duration localCleanerInterval; - private final Duration localCleanTime; - private final Duration backupCleanerInterval; - private final Duration backupCleanTime; - private final ScheduledExecutorService scheduler; - private final ExecutorService backupExecutor; - private final Duration maxCompletedTransactionAge; - - private final AtomicBoolean started = new AtomicBoolean(); - - private final CounterStat transactionJobErrors = new CounterStat(); - private final CounterStat backupJobErrors = new CounterStat(); - private final CounterStat localJobErrors = new CounterStat(); - private final CounterStat backupShardsQueued = new CounterStat(); - private final CounterStat backupShardsCleaned = new CounterStat(); - private final CounterStat localShardsCleaned = new CounterStat(); - - @GuardedBy("this") - private final Map shardsToClean = new HashMap<>(); - - @Inject - public ShardCleaner( - DaoSupplier shardDaoSupplier, - Ticker ticker, - NodeManager nodeManager, - StorageService storageService, - Optional backupStore, - ShardCleanerConfig config) - { - this( - shardDaoSupplier, - nodeManager.getCurrentNode().getNodeIdentifier(), - nodeManager.getCurrentNode().isCoordinator(), - ticker, - storageService, - backupStore, - config.getMaxTransactionAge(), - config.getTransactionCleanerInterval(), - config.getLocalCleanerInterval(), - config.getLocalCleanTime(), - config.getBackupCleanerInterval(), - config.getBackupCleanTime(), - config.getBackupDeletionThreads(), - config.getMaxCompletedTransactionAge()); - } - - public ShardCleaner( - DaoSupplier shardDaoSupplier, - String currentNode, - boolean coordinator, - Ticker ticker, - StorageService storageService, - Optional backupStore, - Duration maxTransactionAge, - Duration transactionCleanerInterval, - Duration localCleanerInterval, - Duration localCleanTime, - Duration backupCleanerInterval, - Duration backupCleanTime, - int backupDeletionThreads, - Duration maxCompletedTransactionAge) - { - this.dao = shardDaoSupplier.onDemand(); - this.currentNode = requireNonNull(currentNode, "currentNode is null"); - this.coordinator = coordinator; - this.ticker = requireNonNull(ticker, "ticker is null"); - this.storageService = requireNonNull(storageService, "storageService is null"); - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.maxTransactionAge = requireNonNull(maxTransactionAge, "maxTransactionAge"); - this.transactionCleanerInterval = requireNonNull(transactionCleanerInterval, "transactionCleanerInterval is null"); - this.localCleanerInterval = requireNonNull(localCleanerInterval, "localCleanerInterval is null"); - this.localCleanTime = requireNonNull(localCleanTime, "localCleanTime is null"); - this.backupCleanerInterval = requireNonNull(backupCleanerInterval, "backupCleanerInterval is null"); - this.backupCleanTime = requireNonNull(backupCleanTime, "backupCleanTime is null"); - this.scheduler = newScheduledThreadPool(2, daemonThreadsNamed("shard-cleaner-%s")); - this.backupExecutor = newFixedThreadPool(backupDeletionThreads, daemonThreadsNamed("shard-cleaner-backup-%s")); - this.maxCompletedTransactionAge = requireNonNull(maxCompletedTransactionAge, "maxCompletedTransactionAge is null"); - } - - @PostConstruct - public void start() - { - if (!started.getAndSet(true)) { - startJobs(); - } - } - - @PreDestroy - public void shutdown() - { - scheduler.shutdownNow(); - backupExecutor.shutdownNow(); - } - - @Managed - @Nested - public CounterStat getTransactionJobErrors() - { - return transactionJobErrors; - } - - @Managed - @Nested - public CounterStat getBackupJobErrors() - { - return backupJobErrors; - } - - @Managed - @Nested - public CounterStat getLocalJobErrors() - { - return localJobErrors; - } - - @Managed - @Nested - public CounterStat getBackupShardsQueued() - { - return backupShardsQueued; - } - - @Managed - @Nested - public CounterStat getBackupShardsCleaned() - { - return backupShardsCleaned; - } - - @Managed - @Nested - public CounterStat getLocalShardsCleaned() - { - return localShardsCleaned; - } - - private void startJobs() - { - if (coordinator) { - startTransactionCleanup(); - if (backupStore.isPresent()) { - scheduleBackupCleanup(); - } - } - - // We can only delete local shards if the backup store is present, - // since there is a race condition between shards getting created - // on a worker and being committed (referenced) in the database. - if (backupStore.isPresent()) { - scheduleLocalCleanup(); - } - } - - private void startTransactionCleanup() - { - scheduler.scheduleWithFixedDelay(() -> { - try { - abortOldTransactions(); - deleteOldCompletedTransactions(); - deleteOldShards(); - } - catch (Throwable t) { - log.error(t, "Error cleaning transactions"); - transactionJobErrors.update(1); - } - }, 0, transactionCleanerInterval.toMillis(), MILLISECONDS); - } - - private void scheduleBackupCleanup() - { - scheduler.scheduleWithFixedDelay(this::runBackupCleanup, 0, backupCleanerInterval.toMillis(), MILLISECONDS); - } - - private void scheduleLocalCleanup() - { - Set local = getLocalShards(); - scheduler.submit(() -> { - waitJitterTime(); - runLocalCleanupImmediately(local); - }); - - scheduler.scheduleWithFixedDelay(() -> { - waitJitterTime(); - runLocalCleanup(); - }, 0, localCleanerInterval.toMillis(), MILLISECONDS); - } - - private synchronized void runBackupCleanup() - { - try { - cleanBackupShards(); - } - catch (Throwable t) { - log.error(t, "Error cleaning backup shards"); - backupJobErrors.update(1); - } - } - - private void waitJitterTime() - { - try { - // jitter to avoid overloading database - long interval = this.localCleanerInterval.roundTo(SECONDS); - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval)); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private synchronized void runLocalCleanupImmediately(Set local) - { - try { - cleanLocalShardsImmediately(local); - } - catch (Throwable t) { - log.error(t, "Error cleaning local shards"); - localJobErrors.update(1); - } - } - - private synchronized void runLocalCleanup() - { - try { - cleanLocalShards(); - } - catch (Throwable t) { - log.error(t, "Error cleaning local shards"); - localJobErrors.update(1); - } - } - - @Managed - public void startBackupCleanup() - { - scheduler.submit(this::runBackupCleanup); - } - - @Managed - public void startLocalCleanup() - { - scheduler.submit(this::runLocalCleanup); - } - - @Managed - public void startLocalCleanupImmediately() - { - scheduler.submit(() -> { - runLocalCleanupImmediately(getLocalShards()); - }); - } - - @VisibleForTesting - void abortOldTransactions() - { - dao.abortOldTransactions(maxTimestamp(maxTransactionAge)); - } - - @VisibleForTesting - void deleteOldShards() - { - while (!Thread.currentThread().isInterrupted()) { - List shards = dao.getOldCreatedShardsBatch(); - if (shards.isEmpty()) { - break; - } - dao.insertDeletedShards(shards); - dao.deleteCreatedShards(shards); - backupShardsQueued.update(shards.size()); - } - } - - @VisibleForTesting - void deleteOldCompletedTransactions() - { - while (!Thread.currentThread().isInterrupted()) { - int deleted = dao.deleteOldCompletedTransactions(maxTimestamp(maxCompletedTransactionAge)); - if (deleted < CLEANUP_TRANSACTIONS_BATCH_SIZE) { - break; - } - } - } - - @VisibleForTesting - synchronized Set getLocalShards() - { - return storageService.getStorageShards(); - } - - @VisibleForTesting - synchronized void cleanLocalShardsImmediately(Set local) - { - // get shards assigned to the local node - Set assigned = dao.getNodeShards(currentNode, null).stream() - .map(ShardMetadata::getShardUuid) - .collect(toSet()); - - // only delete local files that are not assigned - Set deletions = Sets.difference(local, assigned); - - for (UUID uuid : deletions) { - deleteFile(storageService.getStorageFile(uuid)); - } - - localShardsCleaned.update(deletions.size()); - log.info("Cleaned %s local shards immediately", deletions.size()); - } - - @VisibleForTesting - synchronized void cleanLocalShards() - { - // find all files on the local node - Set local = getLocalShards(); - - // get shards assigned to the local node - Set assigned = dao.getNodeShards(currentNode, null).stream() - .map(ShardMetadata::getShardUuid) - .collect(toSet()); - - // un-mark previously marked files that are now assigned - for (UUID uuid : assigned) { - shardsToClean.remove(uuid); - } - - // mark all files that are not assigned - for (UUID uuid : local) { - if (!assigned.contains(uuid)) { - shardsToClean.putIfAbsent(uuid, ticker.read()); - } - } - - // delete files marked earlier than the clean interval - long threshold = ticker.read() - localCleanTime.roundTo(NANOSECONDS); - Set deletions = shardsToClean.entrySet().stream() - .filter(entry -> entry.getValue() < threshold) - .map(Map.Entry::getKey) - .collect(toSet()); - if (deletions.isEmpty()) { - return; - } - - for (UUID uuid : deletions) { - deleteFile(storageService.getStorageFile(uuid)); - shardsToClean.remove(uuid); - } - - localShardsCleaned.update(deletions.size()); - log.info("Cleaned %s local shards", deletions.size()); - } - - @VisibleForTesting - void cleanBackupShards() - { - Set processing = newConcurrentHashSet(); - BlockingQueue completed = new LinkedBlockingQueue<>(); - boolean fill = true; - - while (!Thread.currentThread().isInterrupted()) { - // get a new batch if any completed and we are under the batch size - Set uuids = ImmutableSet.of(); - if (fill && (processing.size() < CLEANABLE_SHARDS_BATCH_SIZE)) { - uuids = dao.getCleanableShardsBatch(maxTimestamp(backupCleanTime)); - fill = false; - } - if (uuids.isEmpty() && processing.isEmpty()) { - break; - } - - // skip any that are already processing and mark remaining as processing - uuids = ImmutableSet.copyOf(difference(uuids, processing)); - processing.addAll(uuids); - - // execute deletes - for (UUID uuid : uuids) { - runAsync(() -> backupStore.get().deleteShard(uuid), backupExecutor) - .thenAccept(v -> completed.add(uuid)) - .whenComplete((v, e) -> { - if (e != null) { - log.error(e, "Error cleaning backup shard: %s", uuid); - backupJobErrors.update(1); - processing.remove(uuid); - } - }); - } - - // get the next batch of completed deletes - int desired = min(100, processing.size()); - Collection done = drain(completed, desired, 100, MILLISECONDS); - if (done.isEmpty()) { - continue; - } - - // remove completed deletes from database - processing.removeAll(done); - dao.deleteCleanedShards(done); - backupShardsCleaned.update(done.size()); - fill = true; - } - } - - private static Collection drain(BlockingQueue queue, int desired, long timeout, TimeUnit unit) - { - long start = System.nanoTime(); - Collection result = new ArrayList<>(); - - while (true) { - queue.drainTo(result); - if (result.size() >= desired) { - return result; - } - - long elapsedNanos = System.nanoTime() - start; - long remainingNanos = unit.toNanos(timeout) - elapsedNanos; - if (remainingNanos <= 0) { - return result; - } - - try { - T value = queue.poll(remainingNanos, NANOSECONDS); - if (value != null) { - result.add(value); - } - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return result; - } - } - } - - private static Timestamp maxTimestamp(Duration duration) - { - return new Timestamp(System.currentTimeMillis() - duration.toMillis()); - } - - @SuppressWarnings("ResultOfMethodCallIgnored") - private static void deleteFile(File file) - { - file.delete(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleanerConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleanerConfig.java deleted file mode 100644 index 9c112b2773df..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleanerConfig.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; -import io.airlift.units.MaxDuration; -import io.airlift.units.MinDuration; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; - -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MINUTES; - -public class ShardCleanerConfig -{ - private Duration maxTransactionAge = new Duration(1, DAYS); - private Duration transactionCleanerInterval = new Duration(10, MINUTES); - private Duration localCleanerInterval = new Duration(1, HOURS); - private Duration localCleanTime = new Duration(4, HOURS); - private Duration backupCleanerInterval = new Duration(5, MINUTES); - private Duration backupCleanTime = new Duration(1, DAYS); - private int backupDeletionThreads = 50; - private Duration maxCompletedTransactionAge = new Duration(1, DAYS); - - @NotNull - @MinDuration("1m") - @MaxDuration("30d") - public Duration getMaxTransactionAge() - { - return maxTransactionAge; - } - - @Config("raptor.max-transaction-age") - @ConfigDescription("Maximum time a transaction may run before it is aborted") - public ShardCleanerConfig setMaxTransactionAge(Duration maxTransactionAge) - { - this.maxTransactionAge = maxTransactionAge; - return this; - } - - @NotNull - @MinDuration("1m") - public Duration getTransactionCleanerInterval() - { - return transactionCleanerInterval; - } - - @Config("raptor.transaction-cleaner-interval") - @ConfigDescription("How often to cleanup expired transactions") - public ShardCleanerConfig setTransactionCleanerInterval(Duration transactionCleanerInterval) - { - this.transactionCleanerInterval = transactionCleanerInterval; - return this; - } - - @NotNull - @MinDuration("1m") - public Duration getLocalCleanerInterval() - { - return localCleanerInterval; - } - - @Config("raptor.local-cleaner-interval") - @ConfigDescription("How often to discover local shards that need to be cleaned up") - public ShardCleanerConfig setLocalCleanerInterval(Duration localCleanerInterval) - { - this.localCleanerInterval = localCleanerInterval; - return this; - } - - @NotNull - public Duration getLocalCleanTime() - { - return localCleanTime; - } - - @Config("raptor.local-clean-time") - @ConfigDescription("How long to wait after discovery before cleaning local shards") - public ShardCleanerConfig setLocalCleanTime(Duration localCleanTime) - { - this.localCleanTime = localCleanTime; - return this; - } - - @NotNull - @MinDuration("1m") - public Duration getBackupCleanerInterval() - { - return backupCleanerInterval; - } - - @Config("raptor.backup-cleaner-interval") - @ConfigDescription("How often to check for backup shards that need to be cleaned up") - public ShardCleanerConfig setBackupCleanerInterval(Duration backupCleanerInterval) - { - this.backupCleanerInterval = backupCleanerInterval; - return this; - } - - @NotNull - public Duration getBackupCleanTime() - { - return backupCleanTime; - } - - @Config("raptor.backup-clean-time") - @ConfigDescription("How long to wait after deletion before cleaning backup shards") - public ShardCleanerConfig setBackupCleanTime(Duration backupCleanTime) - { - this.backupCleanTime = backupCleanTime; - return this; - } - - @Min(1) - public int getBackupDeletionThreads() - { - return backupDeletionThreads; - } - - @Config("raptor.backup-deletion-threads") - @ConfigDescription("Maximum number of threads to use for deleting shards from backup store") - public ShardCleanerConfig setBackupDeletionThreads(int backupDeletionThreads) - { - this.backupDeletionThreads = backupDeletionThreads; - return this; - } - - @NotNull - @MinDuration("1m") - @MaxDuration("30d") - public Duration getMaxCompletedTransactionAge() - { - return maxCompletedTransactionAge; - } - - @Config("raptor.max-completed-transaction-age") - @ConfigDescription("Maximum time a record of a successful or failed transaction is kept") - public ShardCleanerConfig setMaxCompletedTransactionAge(Duration maxCompletedTransactionAge) - { - this.maxCompletedTransactionAge = maxCompletedTransactionAge; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDao.java deleted file mode 100644 index f5ac3c33d94b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDao.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.plugin.raptor.legacy.util.UuidUtil.UuidArgumentFactory; -import io.trino.plugin.raptor.legacy.util.UuidUtil.UuidColumnMapper; -import org.jdbi.v3.sqlobject.config.RegisterArgumentFactory; -import org.jdbi.v3.sqlobject.config.RegisterColumnMapper; -import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; -import org.jdbi.v3.sqlobject.config.RegisterRowMapper; -import org.jdbi.v3.sqlobject.customizer.Bind; -import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; -import org.jdbi.v3.sqlobject.statement.SqlBatch; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.sql.Timestamp; -import java.util.List; -import java.util.Set; -import java.util.UUID; - -@RegisterArgumentFactory(UuidArgumentFactory.class) -@RegisterColumnMapper(UuidColumnMapper.class) -@RegisterConstructorMapper(BucketNode.class) -@RegisterConstructorMapper(NodeSize.class) -@RegisterConstructorMapper(RaptorNode.class) -@RegisterRowMapper(ShardMetadata.Mapper.class) -public interface ShardDao -{ - int CLEANABLE_SHARDS_BATCH_SIZE = 1000; - int CLEANUP_TRANSACTIONS_BATCH_SIZE = 10_000; - - String SHARD_METADATA_COLUMNS = "table_id, shard_id, shard_uuid, bucket_number, row_count, compressed_size, uncompressed_size, xxhash64"; - - @SqlUpdate("INSERT INTO nodes (node_identifier) VALUES (:nodeIdentifier)") - @GetGeneratedKeys - int insertNode(String nodeIdentifier); - - @SqlUpdate("INSERT INTO shard_nodes (shard_id, node_id)\n" + - "VALUES ((SELECT shard_id FROM shards WHERE shard_uuid = :shardUuid), :nodeId)") - void insertShardNode(UUID shardUuid, int nodeId); - - @SqlBatch("DELETE FROM shard_nodes\n" + - "WHERE shard_id = (SELECT shard_id FROM shards WHERE shard_uuid = :shardUuid)\n" + - " AND node_id = :nodeId") - void deleteShardNodes(UUID shardUuid, @Bind("nodeId") Iterable nodeIds); - - @SqlQuery("SELECT node_id FROM nodes WHERE node_identifier = :nodeIdentifier") - Integer getNodeId(String nodeIdentifier); - - @SqlQuery("SELECT node_identifier FROM nodes WHERE node_id = :nodeId") - String getNodeIdentifier(int nodeId); - - @SqlQuery("SELECT node_id, node_identifier FROM nodes") - List getNodes(); - - @SqlQuery("SELECT " + SHARD_METADATA_COLUMNS + " FROM shards WHERE shard_uuid = :shardUuid") - ShardMetadata getShard(UUID shardUuid); - - @SqlQuery("SELECT " + SHARD_METADATA_COLUMNS + "\n" + - "FROM (\n" + - " SELECT s.*\n" + - " FROM shards s\n" + - " JOIN shard_nodes sn ON (s.shard_id = sn.shard_id)\n" + - " JOIN nodes n ON (sn.node_id = n.node_id)\n" + - " WHERE n.node_identifier = :nodeIdentifier\n" + - " AND s.bucket_number IS NULL\n" + - " AND (s.table_id = :tableId OR :tableId IS NULL)\n" + - " UNION ALL\n" + - " SELECT s.*\n" + - " FROM shards s\n" + - " JOIN tables t ON (s.table_id = t.table_id)\n" + - " JOIN distributions d ON (t.distribution_id = d.distribution_id)\n" + - " JOIN buckets b ON (\n" + - " d.distribution_id = b.distribution_id AND\n" + - " s.bucket_number = b.bucket_number)\n" + - " JOIN nodes n ON (b.node_id = n.node_id)\n" + - " WHERE n.node_identifier = :nodeIdentifier\n" + - " AND (s.table_id = :tableId OR :tableId IS NULL)\n" + - ") x") - Set getNodeShards(String nodeIdentifier, Long tableId); - - @SqlQuery("SELECT n.node_identifier, x.bytes\n" + - "FROM (\n" + - " SELECT node_id, sum(compressed_size) bytes\n" + - " FROM (\n" + - " SELECT sn.node_id, s.compressed_size\n" + - " FROM shards s\n" + - " JOIN shard_nodes sn ON (s.shard_id = sn.shard_id)\n" + - " WHERE s.bucket_number IS NULL\n" + - " UNION ALL\n" + - " SELECT b.node_id, s.compressed_size\n" + - " FROM shards s\n" + - " JOIN tables t ON (s.table_id = t.table_id)\n" + - " JOIN distributions d ON (t.distribution_id = d.distribution_id)\n" + - " JOIN buckets b ON (\n" + - " d.distribution_id = b.distribution_id AND\n" + - " s.bucket_number = b.bucket_number)\n" + - " ) x\n" + - " GROUP BY node_id\n" + - ") x\n" + - "JOIN nodes n ON (x.node_id = n.node_id)") - Set getNodeSizes(); - - @SqlUpdate("DELETE FROM shard_nodes WHERE shard_id IN (\n" + - " SELECT shard_id\n" + - " FROM shards\n" + - " WHERE table_id = :tableId)") - void dropShardNodes(long tableId); - - @SqlUpdate("DELETE FROM shards WHERE table_id = :tableId") - void dropShards(long tableId); - - @SqlUpdate("INSERT INTO external_batches (external_batch_id, successful)\n" + - "VALUES (:externalBatchId, TRUE)") - void insertExternalBatch(String externalBatchId); - - @SqlQuery("SELECT count(*)\n" + - "FROM external_batches\n" + - "WHERE external_batch_id = :externalBatchId") - boolean externalBatchExists(String externalBatchId); - - @SqlUpdate("INSERT INTO transactions (start_time) VALUES (CURRENT_TIMESTAMP)") - @GetGeneratedKeys - long insertTransaction(); - - @SqlUpdate("UPDATE transactions SET\n" + - " successful = :successful\n" + - ", end_time = CURRENT_TIMESTAMP\n" + - "WHERE transaction_id = :transactionId\n" + - " AND successful IS NULL") - int finalizeTransaction( - long transactionId, - boolean successful); - - @SqlQuery("SELECT successful FROM transactions WHERE transaction_id = :transactionId") - Boolean transactionSuccessful(long transactionId); - - @SqlUpdate("UPDATE transactions SET\n" + - " successful = FALSE\n" + - ", end_time = CURRENT_TIMESTAMP\n" + - "WHERE successful IS NULL\n" + - " AND start_time < :maxStartTime") - void abortOldTransactions(Timestamp maxStartTime); - - @SqlUpdate("INSERT INTO created_shards (shard_uuid, transaction_id)\n" + - "VALUES (:shardUuid, :transactionId)") - void insertCreatedShard( - UUID shardUuid, - long transactionId); - - @SqlUpdate("DELETE FROM created_shards WHERE transaction_id = :transactionId") - void deleteCreatedShards(long transactionId); - - @SqlBatch("DELETE FROM created_shards WHERE shard_uuid = :shardUuid") - void deleteCreatedShards(@Bind("shardUuid") Iterable shardUuids); - - default void insertDeletedShards(Iterable shardUuids) - { - throw new UnsupportedOperationException(); - } - - @SqlUpdate("INSERT INTO deleted_shards (shard_uuid, delete_time)\n" + - "SELECT shard_uuid, CURRENT_TIMESTAMP\n" + - "FROM shards\n" + - "WHERE table_id = :tableId") - void insertDeletedShards(long tableId); - - @SqlQuery("SELECT s.shard_uuid\n" + - "FROM created_shards s\n" + - "JOIN transactions t ON (s.transaction_id = t.transaction_id)\n" + - "WHERE NOT t.successful\n" + - "LIMIT 10000") - List getOldCreatedShardsBatch(); - - @SqlQuery("SELECT shard_uuid\n" + - "FROM deleted_shards\n" + - "WHERE delete_time < :maxDeleteTime\n" + - "LIMIT " + CLEANABLE_SHARDS_BATCH_SIZE) - Set getCleanableShardsBatch(Timestamp maxDeleteTime); - - @SqlBatch("DELETE FROM deleted_shards WHERE shard_uuid = :shardUuid") - void deleteCleanedShards(@Bind("shardUuid") Iterable shardUuids); - - @SqlBatch("INSERT INTO buckets (distribution_id, bucket_number, node_id)\n" + - "VALUES (:distributionId, :bucketNumber, :nodeId)\n") - void insertBuckets( - long distributionId, - @Bind("bucketNumber") List bucketNumbers, - @Bind("nodeId") List nodeIds); - - @SqlQuery("SELECT b.bucket_number, n.node_identifier\n" + - "FROM buckets b\n" + - "JOIN nodes n ON (b.node_id = n.node_id)\n" + - "WHERE b.distribution_id = :distributionId\n" + - "ORDER BY b.bucket_number") - List getBucketNodes(long distributionId); - - @SqlQuery("SELECT distribution_id, distribution_name, column_types, bucket_count\n" + - "FROM distributions\n" + - "WHERE distribution_id IN (SELECT distribution_id FROM tables)") - List listActiveDistributions(); - - @SqlQuery("SELECT SUM(compressed_size)\n" + - "FROM tables\n" + - "WHERE distribution_id = :distributionId") - long getDistributionSizeBytes(long distributionId); - - @SqlUpdate("UPDATE buckets SET node_id = :nodeId\n" + - "WHERE distribution_id = :distributionId\n" + - " AND bucket_number = :bucketNumber") - void updateBucketNode( - long distributionId, - int bucketNumber, - int nodeId); - - default int deleteOldCompletedTransactions(Timestamp maxEndTime) - { - throw new UnsupportedOperationException(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDelta.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDelta.java deleted file mode 100644 index e53a37f58bc1..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDelta.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardDelta -{ - private final List oldShardUuids; - private final List newShards; - - @JsonCreator - public ShardDelta( - @JsonProperty("oldShardUuids") List oldShardUuids, - @JsonProperty("newShards") List newShards) - { - this.oldShardUuids = ImmutableList.copyOf(requireNonNull(oldShardUuids, "oldShardUuids is null")); - this.newShards = ImmutableList.copyOf(requireNonNull(newShards, "newShards is null")); - } - - @JsonProperty - public List getOldShardUuids() - { - return oldShardUuids; - } - - @JsonProperty - public List getNewShards() - { - return newShards; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("oldShardUuids", oldShardUuids) - .add("newShards", newShards) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardInfo.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardInfo.java deleted file mode 100644 index 5050c376b69b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardInfo.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; - -import java.util.List; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public class ShardInfo -{ - private final UUID shardUuid; - private final OptionalInt bucketNumber; - private final Set nodeIdentifiers; - private final List columnStats; - private final long rowCount; - private final long compressedSize; - private final long uncompressedSize; - private final long xxhash64; - - @JsonCreator - public ShardInfo( - @JsonProperty("shardUuid") UUID shardUuid, - @JsonProperty("bucketNumber") OptionalInt bucketNumber, - @JsonProperty("nodeIdentifiers") Set nodeIdentifiers, - @JsonProperty("columnStats") List columnStats, - @JsonProperty("rowCount") long rowCount, - @JsonProperty("compressedSize") long compressedSize, - @JsonProperty("uncompressedSize") long uncompressedSize, - @JsonProperty("xxhash64") long xxhash64) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.nodeIdentifiers = ImmutableSet.copyOf(requireNonNull(nodeIdentifiers, "nodeIdentifiers is null")); - this.columnStats = ImmutableList.copyOf(requireNonNull(columnStats, "columnStats is null")); - - checkArgument(rowCount >= 0, "rowCount must be positive"); - checkArgument(compressedSize >= 0, "compressedSize must be positive"); - checkArgument(uncompressedSize >= 0, "uncompressedSize must be positive"); - this.rowCount = rowCount; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - - this.xxhash64 = xxhash64; - } - - @JsonProperty - public UUID getShardUuid() - { - return shardUuid; - } - - @JsonProperty - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - @JsonProperty - public Set getNodeIdentifiers() - { - return nodeIdentifiers; - } - - @JsonProperty - public List getColumnStats() - { - return columnStats; - } - - @JsonProperty - public long getRowCount() - { - return rowCount; - } - - @JsonProperty - public long getCompressedSize() - { - return compressedSize; - } - - @JsonProperty - public long getUncompressedSize() - { - return uncompressedSize; - } - - @JsonProperty - public long getXxhash64() - { - return xxhash64; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuid", shardUuid) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .add("nodeIdentifiers", nodeIdentifiers) - .add("columnStats", columnStats) - .add("rowCount", rowCount) - .add("compressedSize", compressedSize) - .add("uncompressedSize", uncompressedSize) - .add("xxhash64", format("%016x", xxhash64)) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardIterator.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardIterator.java deleted file mode 100644 index 22fcab27f5df..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardIterator.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.AbstractIterator; -import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.TrinoException; -import io.trino.spi.predicate.TupleDomain; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.result.ResultIterator; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; -import java.util.function.Function; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.util.ArrayUtil.intArrayFromBytes; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.enableStreamingResults; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static java.lang.String.format; -import static java.util.stream.Collectors.toSet; - -final class ShardIterator - extends AbstractIterator - implements ResultIterator -{ - private static final Logger log = Logger.get(ShardIterator.class); - private final Map nodeMap = new HashMap<>(); - - private final boolean merged; - private final List bucketToNode; - private final ShardDao dao; - private final Connection connection; - private final PreparedStatement statement; - private final ResultSet resultSet; - private boolean first = true; - - public ShardIterator( - long tableId, - boolean merged, - Optional> bucketToNode, - TupleDomain effectivePredicate, - Jdbi dbi) - { - this.merged = merged; - this.bucketToNode = bucketToNode.orElse(null); - ShardPredicate predicate = ShardPredicate.create(effectivePredicate); - - String sql; - if (bucketToNode.isPresent()) { - sql = "SELECT shard_uuid, bucket_number FROM %s WHERE %s ORDER BY bucket_number"; - } - else { - sql = "SELECT shard_uuid, node_ids FROM %s WHERE %s"; - } - sql = format(sql, shardIndexTable(tableId), predicate.getPredicate()); - - dao = onDemandDao(dbi, ShardDao.class); - fetchNodes(); - - try { - connection = dbi.open().getConnection(); - statement = connection.prepareStatement(sql); - enableStreamingResults(statement); - predicate.bind(statement); - log.debug("Running query: %s", statement); - resultSet = statement.executeQuery(); - } - catch (SQLException e) { - close(); - throw metadataError(e); - } - catch (Throwable t) { - close(); - throw t; - } - } - - @Override - protected BucketShards computeNext() - { - try { - return merged ? computeMerged() : compute(); - } - catch (SQLException e) { - throw metadataError(e); - } - } - - @SuppressWarnings({"UnusedDeclaration", "EmptyTryBlock"}) - @Override - public void close() - { - // use try-with-resources to close everything properly - try (Connection connection = this.connection; - Statement statement = this.statement; - ResultSet resultSet = this.resultSet) { - // do nothing - } - catch (SQLException _) { - } - } - - @Override - public StatementContext getContext() - { - throw new UnsupportedOperationException(); - } - - /** - * Compute split-per-shard (separate split for each shard). - */ - private BucketShards compute() - throws SQLException - { - if (!resultSet.next()) { - return endOfData(); - } - - UUID shardUuid = uuidFromBytes(resultSet.getBytes("shard_uuid")); - Set nodeIdentifiers; - OptionalInt bucketNumber = OptionalInt.empty(); - - if (bucketToNode != null) { - int bucket = resultSet.getInt("bucket_number"); - bucketNumber = OptionalInt.of(bucket); - nodeIdentifiers = ImmutableSet.of(getBucketNode(bucket)); - } - else { - List nodeIds = intArrayFromBytes(resultSet.getBytes("node_ids")); - nodeIdentifiers = getNodeIdentifiers(nodeIds, shardUuid); - } - - ShardNodes shard = new ShardNodes(shardUuid, nodeIdentifiers); - return new BucketShards(bucketNumber, ImmutableSet.of(shard)); - } - - /** - * Compute split-per-bucket (single split for all shards in a bucket). - */ - private BucketShards computeMerged() - throws SQLException - { - if (resultSet.isAfterLast()) { - return endOfData(); - } - if (first) { - first = false; - if (!resultSet.next()) { - return endOfData(); - } - } - - int bucketNumber = resultSet.getInt("bucket_number"); - ImmutableSet.Builder shards = ImmutableSet.builder(); - - do { - UUID shardUuid = uuidFromBytes(resultSet.getBytes("shard_uuid")); - int bucket = resultSet.getInt("bucket_number"); - Set nodeIdentifiers = ImmutableSet.of(getBucketNode(bucket)); - - shards.add(new ShardNodes(shardUuid, nodeIdentifiers)); - } - while (resultSet.next() && resultSet.getInt("bucket_number") == bucketNumber); - - return new BucketShards(OptionalInt.of(bucketNumber), shards.build()); - } - - private String getBucketNode(int bucket) - { - String node = bucketToNode.get(bucket); - if (node == null) { - throw new TrinoException(RAPTOR_ERROR, "No node mapping for bucket: " + bucket); - } - return node; - } - - private Set getNodeIdentifiers(List nodeIds, UUID shardUuid) - { - Function fetchNode = id -> fetchNode(id, shardUuid); - return nodeIds.stream() - .map(id -> nodeMap.computeIfAbsent(id, fetchNode)) - .collect(toSet()); - } - - private String fetchNode(int id, UUID shardUuid) - { - String node = dao.getNodeIdentifier(id); - if (node == null) { - throw new TrinoException(RAPTOR_ERROR, format("Missing node ID [%s] for shard: %s", id, shardUuid)); - } - return node; - } - - private void fetchNodes() - { - for (RaptorNode node : dao.getNodes()) { - nodeMap.put(node.getNodeId(), node.getNodeIdentifier()); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardManager.java deleted file mode 100644 index 13f18ee7dcd0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardManager.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.predicate.TupleDomain; -import org.jdbi.v3.core.result.ResultIterator; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -public interface ShardManager -{ - /** - * Create a table. - */ - void createTable(long tableId, List columns, boolean bucketed, OptionalLong temporalColumnId); - - /** - * Drop a table. - */ - void dropTable(long tableId); - - /** - * Add a column to the end of the table. - */ - void addColumn(long tableId, ColumnInfo column); - - /** - * Commit data for a table. - */ - void commitShards(long transactionId, long tableId, List columns, Collection shards, Optional externalBatchId, long updateTime); - - /** - * Replace oldShardsUuids with newShards. - */ - void replaceShardUuids(long transactionId, long tableId, List columns, Set oldShardUuids, Collection newShards, OptionalLong updateTime); - - /** - * Get shard metadata for a shard. - */ - ShardMetadata getShard(UUID shardUuid); - - /** - * Get shard metadata for shards on a given node. - */ - Set getNodeShards(String nodeIdentifier); - - /** - * Get shard metadata for shards on a given node. - */ - Set getNodeShards(String nodeIdentifier, long tableId); - - /** - * Return the shard nodes for a non-bucketed table. - */ - ResultIterator getShardNodes(long tableId, TupleDomain effectivePredicate); - - /** - * Return the shard nodes for a bucketed table. - */ - ResultIterator getShardNodesBucketed(long tableId, boolean merged, List bucketToNode, TupleDomain effectivePredicate); - - /** - * Remove all old shard assignments and assign a shard to a node - */ - void replaceShardAssignment(long tableId, UUID shardUuid, String nodeIdentifier, boolean gracePeriod); - - /** - * Get the number of bytes used by assigned shards per node. - */ - Map getNodeBytes(); - - /** - * Begin a transaction for creating shards. - * - * @return transaction ID - */ - long beginTransaction(); - - /** - * Rollback a transaction. - */ - void rollbackTransaction(long transactionId); - - /** - * Create initial bucket assignments for a distribution. - */ - void createBuckets(long distributionId, int bucketCount); - - /** - * Get map of buckets to node identifiers for a distribution. - */ - List getBucketAssignments(long distributionId); - - /** - * Change the node a bucket is assigned to. - */ - void updateBucketAssignment(long distributionId, int bucketNumber, String nodeId); - - /** - * Get all active distributions. - */ - List getDistributions(); - - /** - * Get total physical size of all tables in a distribution. - */ - long getDistributionSizeInBytes(long distributionId); - - /** - * Get list of bucket nodes for a distribution. - */ - List getBucketNodes(long distributionId); - - /** - * Return the subset of shard uuids that exist - */ - Set getExistingShardUuids(long tableId, Set shardUuids); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardMetadata.java deleted file mode 100644 index fd0d4c35bac6..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardMetadata.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.units.DataSize; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Objects; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.ToStringHelper; -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalInt; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalLong; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public class ShardMetadata -{ - private final long tableId; - private final long shardId; - private final UUID shardUuid; - private final OptionalInt bucketNumber; - private final long rowCount; - private final long compressedSize; - private final long uncompressedSize; - private final OptionalLong xxhash64; - private final OptionalLong rangeStart; - private final OptionalLong rangeEnd; - - public ShardMetadata( - long tableId, - long shardId, - UUID shardUuid, - OptionalInt bucketNumber, - long rowCount, - long compressedSize, - long uncompressedSize, - OptionalLong xxhash64, - OptionalLong rangeStart, - OptionalLong rangeEnd) - { - checkArgument(tableId > 0, "tableId must be > 0"); - checkArgument(shardId > 0, "shardId must be > 0"); - checkArgument(rowCount >= 0, "rowCount must be >= 0"); - checkArgument(compressedSize >= 0, "compressedSize must be >= 0"); - checkArgument(uncompressedSize >= 0, "uncompressedSize must be >= 0"); - - this.tableId = tableId; - this.shardId = shardId; - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.rowCount = rowCount; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - this.xxhash64 = requireNonNull(xxhash64, "xxhash64 is null"); - this.rangeStart = requireNonNull(rangeStart, "rangeStart is null"); - this.rangeEnd = requireNonNull(rangeEnd, "rangeEnd is null"); - } - - public long getTableId() - { - return tableId; - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public long getShardId() - { - return shardId; - } - - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - public long getRowCount() - { - return rowCount; - } - - public long getCompressedSize() - { - return compressedSize; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - - public OptionalLong getXxhash64() - { - return xxhash64; - } - - public OptionalLong getRangeStart() - { - return rangeStart; - } - - public OptionalLong getRangeEnd() - { - return rangeEnd; - } - - public ShardMetadata withTimeRange(long rangeStart, long rangeEnd) - { - return new ShardMetadata( - tableId, - shardId, - shardUuid, - bucketNumber, - rowCount, - compressedSize, - uncompressedSize, - xxhash64, - OptionalLong.of(rangeStart), - OptionalLong.of(rangeEnd)); - } - - @Override - public String toString() - { - ToStringHelper stringHelper = toStringHelper(this) - .add("tableId", tableId) - .add("shardId", shardId) - .add("shardUuid", shardUuid) - .add("rowCount", rowCount) - .add("compressedSize", DataSize.succinctBytes(compressedSize)) - .add("uncompressedSize", DataSize.succinctBytes(uncompressedSize)); - - if (bucketNumber.isPresent()) { - stringHelper.add("bucketNumber", bucketNumber.getAsInt()); - } - if (xxhash64.isPresent()) { - stringHelper.add("xxhash64", format("%16x", xxhash64.getAsLong())); - } - if (rangeStart.isPresent()) { - stringHelper.add("rangeStart", rangeStart.getAsLong()); - } - if (rangeEnd.isPresent()) { - stringHelper.add("rangeEnd", rangeEnd.getAsLong()); - } - return stringHelper.toString(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ShardMetadata that = (ShardMetadata) o; - return tableId == that.tableId && - shardId == that.shardId && - Objects.equals(bucketNumber, that.bucketNumber) && - rowCount == that.rowCount && - compressedSize == that.compressedSize && - uncompressedSize == that.uncompressedSize && - Objects.equals(xxhash64, that.xxhash64) && - Objects.equals(shardUuid, that.shardUuid) && - Objects.equals(rangeStart, that.rangeStart) && - Objects.equals(rangeEnd, that.rangeEnd); - } - - @Override - public int hashCode() - { - return Objects.hash( - tableId, - shardId, - shardUuid, - bucketNumber, - rowCount, - compressedSize, - uncompressedSize, - xxhash64, - rangeStart, - rangeEnd); - } - - public static class Mapper - implements RowMapper - { - @Override - public ShardMetadata map(ResultSet r, StatementContext ctx) - throws SQLException - { - return new ShardMetadata( - r.getLong("table_id"), - r.getLong("shard_id"), - uuidFromBytes(r.getBytes("shard_uuid")), - getOptionalInt(r, "bucket_number"), - r.getLong("row_count"), - r.getLong("compressed_size"), - r.getLong("uncompressed_size"), - getOptionalLong(r, "xxhash64"), - OptionalLong.empty(), - OptionalLong.empty()); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardNodes.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardNodes.java deleted file mode 100644 index abfc2f8e2732..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardNodes.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableSet; - -import java.util.Objects; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardNodes -{ - private final UUID shardUuid; - private final Set nodeIdentifiers; - - public ShardNodes(UUID shardUuid, Set nodeIdentifiers) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.nodeIdentifiers = ImmutableSet.copyOf(requireNonNull(nodeIdentifiers, "nodeIdentifiers is null")); - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public Set getNodeIdentifiers() - { - return nodeIdentifiers; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - ShardNodes other = (ShardNodes) obj; - return Objects.equals(this.shardUuid, other.shardUuid) && - Objects.equals(this.nodeIdentifiers, other.nodeIdentifiers); - } - - @Override - public int hashCode() - { - return Objects.hash(shardUuid, nodeIdentifiers); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuid", shardUuid) - .add("nodeIdentifiers", nodeIdentifiers) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardPredicate.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardPredicate.java deleted file mode 100644 index 881b585fab11..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardPredicate.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.TrinoException; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.Ranges; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; - -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.List; -import java.util.Map.Entry; -import java.util.StringJoiner; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.storage.ColumnIndexStatsUtils.jdbcType; -import static io.trino.plugin.raptor.legacy.storage.ShardStats.truncateIndexValue; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidStringToBytes; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -class ShardPredicate -{ - private final String predicate; - private final List types; - private final List values; - private static final int MAX_RANGE_COUNT = 100; - - private ShardPredicate(String predicate, List types, List values) - { - this.predicate = requireNonNull(predicate, "predicate is null"); - this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); - this.values = ImmutableList.copyOf(requireNonNull(values, "values is null")); - checkArgument(types.size() == values.size(), "types and values sizes do not match"); - } - - public String getPredicate() - { - return predicate; - } - - public void bind(PreparedStatement statement) - throws SQLException - { - for (int i = 0; i < types.size(); i++) { - JDBCType type = types.get(i); - Object value = values.get(i); - bindValue(statement, type, value, i + 1); - } - } - - @Override - public String toString() - { - return toStringHelper(this) - .addValue(predicate) - .toString(); - } - - public static ShardPredicate create(TupleDomain tupleDomain) - { - StringJoiner predicate = new StringJoiner(" AND ").setEmptyValue("true"); - ImmutableList.Builder types = ImmutableList.builder(); - ImmutableList.Builder values = ImmutableList.builder(); - - for (Entry entry : tupleDomain.getDomains().get().entrySet()) { - Domain domain = entry.getValue(); - if (domain.isNullAllowed() || domain.isAll()) { - continue; - } - RaptorColumnHandle handle = entry.getKey(); - Type type = handle.getColumnType(); - - JDBCType jdbcType = jdbcType(type); - if (jdbcType == null) { - continue; - } - - if (handle.isShardUuid()) { - predicate.add(createShardPredicate(types, values, domain, jdbcType)); - continue; - } - - if (!domain.getType().isOrderable()) { - continue; - } - - StringJoiner columnPredicate = new StringJoiner(" OR ", "(", ")").setEmptyValue("true"); - Ranges ranges = domain.getValues().getRanges(); - - // prevent generating complicated metadata queries - if (ranges.getRangeCount() > MAX_RANGE_COUNT) { - continue; - } - - for (Range range : ranges.getOrderedRanges()) { - String min; - String max; - if (handle.isBucketNumber()) { - min = "bucket_number"; - max = "bucket_number"; - } - else { - min = minColumn(handle.getColumnId()); - max = maxColumn(handle.getColumnId()); - } - - StringJoiner rangePredicate = new StringJoiner(" AND ", "(", ")").setEmptyValue("true"); - if (!range.isLowUnbounded()) { - rangePredicate.add(format("(%s >= ? OR %s IS NULL)", max, max)); - types.add(jdbcType); - values.add(range.getLowBoundedValue()); - } - if (!range.isHighUnbounded()) { - rangePredicate.add(format("(%s <= ? OR %s IS NULL)", min, min)); - types.add(jdbcType); - values.add(range.getHighBoundedValue()); - } - columnPredicate.add(rangePredicate.toString()); - } - predicate.add(columnPredicate.toString()); - } - return new ShardPredicate(predicate.toString(), types.build(), values.build()); - } - - private static String createShardPredicate(ImmutableList.Builder types, ImmutableList.Builder values, Domain domain, JDBCType jdbcType) - { - List ranges = domain.getValues().getRanges().getOrderedRanges(); - - // only apply predicates if all ranges are single values - if (ranges.isEmpty() || !ranges.stream().allMatch(Range::isSingleValue)) { - return "true"; - } - - ImmutableList.Builder valuesBuilder = ImmutableList.builder(); - ImmutableList.Builder typesBuilder = ImmutableList.builder(); - - StringJoiner rangePredicate = new StringJoiner(" OR "); - for (Range range : ranges) { - Slice uuidText = (Slice) range.getSingleValue(); - try { - Slice uuidBytes = uuidStringToBytes(uuidText); - typesBuilder.add(jdbcType); - valuesBuilder.add(uuidBytes); - } - catch (IllegalArgumentException e) { - return "true"; - } - rangePredicate.add("shard_uuid = ?"); - } - - types.addAll(typesBuilder.build()); - values.addAll(valuesBuilder.build()); - return rangePredicate.toString(); - } - - @VisibleForTesting - protected List getTypes() - { - return types; - } - - @VisibleForTesting - protected List getValues() - { - return values; - } - - public static void bindValue(PreparedStatement statement, JDBCType type, Object value, int index) - throws SQLException - { - if (value == null) { - statement.setNull(index, type.getVendorTypeNumber()); - return; - } - - switch (type) { - case BOOLEAN: - statement.setBoolean(index, (boolean) value); - return; - case INTEGER: - statement.setInt(index, ((Number) value).intValue()); - return; - case BIGINT: - statement.setLong(index, ((Number) value).longValue()); - return; - case DOUBLE: - statement.setDouble(index, ((Number) value).doubleValue()); - return; - case VARBINARY: - statement.setBytes(index, truncateIndexValue((Slice) value).getBytes()); - return; - default: - throw new TrinoException(GENERIC_INTERNAL_ERROR, "Unhandled type: " + type); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardRecorder.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardRecorder.java deleted file mode 100644 index e7c0661edd32..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardRecorder.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import java.util.UUID; - -public interface ShardRecorder -{ - void recordCreatedShard(long transactionId, UUID shardUuid); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Table.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Table.java deleted file mode 100644 index 90ee80608f02..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Table.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getBoxedLong; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalInt; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalLong; -import static java.util.Objects.requireNonNull; - -public final class Table -{ - private final long tableId; - private final Optional distributionId; - private final Optional distributionName; - private final OptionalInt bucketCount; - private final OptionalLong temporalColumnId; - private final boolean organized; - - public Table( - long tableId, - Optional distributionId, - Optional distributionName, - OptionalInt bucketCount, - OptionalLong temporalColumnId, - boolean organized) - { - this.tableId = tableId; - this.distributionId = requireNonNull(distributionId, "distributionId is null"); - this.distributionName = requireNonNull(distributionName, "distributionName is null"); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.temporalColumnId = requireNonNull(temporalColumnId, "temporalColumnId is null"); - this.organized = organized; - } - - public long getTableId() - { - return tableId; - } - - public Optional getDistributionId() - { - return distributionId; - } - - public Optional getDistributionName() - { - return distributionName; - } - - public OptionalInt getBucketCount() - { - return bucketCount; - } - - public OptionalLong getTemporalColumnId() - { - return temporalColumnId; - } - - public boolean isOrganized() - { - return organized; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("tableId", tableId) - .add("distributionId", distributionId.orElse(null)) - .add("bucketCount", bucketCount.isPresent() ? bucketCount.getAsInt() : null) - .add("temporalColumnId", temporalColumnId.isPresent() ? temporalColumnId.getAsLong() : null) - .add("organized", organized) - .omitNullValues() - .toString(); - } - - public static class TableMapper - implements RowMapper - { - @Override - public Table map(ResultSet r, StatementContext ctx) - throws SQLException - { - return new Table( - r.getLong("table_id"), - Optional.ofNullable(getBoxedLong(r, "distribution_id")), - Optional.ofNullable(r.getString("distribution_name")), - getOptionalInt(r, "bucket_count"), - getOptionalLong(r, "temporal_column_id"), - r.getBoolean("organization_enabled")); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableColumn.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableColumn.java deleted file mode 100644 index af07d86df157..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableColumn.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.inject.Inject; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeManager; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.OptionalInt; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalInt; -import static java.util.Objects.requireNonNull; - -public class TableColumn -{ - private final SchemaTableName table; - private final String columnName; - private final Type dataType; - private final long columnId; - private final int ordinalPosition; - private final OptionalInt bucketOrdinal; - private final OptionalInt sortOrdinal; - private final boolean temporal; - - public TableColumn(SchemaTableName table, String columnName, Type dataType, long columnId, int ordinalPosition, OptionalInt bucketOrdinal, OptionalInt sortOrdinal, boolean temporal) - { - this.table = requireNonNull(table, "table is null"); - this.columnName = requireNonNull(columnName, "columnName is null"); - this.dataType = requireNonNull(dataType, "dataType is null"); - this.columnId = columnId; - this.ordinalPosition = ordinalPosition; - this.bucketOrdinal = requireNonNull(bucketOrdinal, "bucketOrdinal is null"); - this.sortOrdinal = requireNonNull(sortOrdinal, "sortOrdinal is null"); - this.temporal = temporal; - } - - public SchemaTableName getTable() - { - return table; - } - - public String getColumnName() - { - return columnName; - } - - public Type getDataType() - { - return dataType; - } - - public long getColumnId() - { - return columnId; - } - - public int getOrdinalPosition() - { - return ordinalPosition; - } - - public OptionalInt getBucketOrdinal() - { - return bucketOrdinal; - } - - public OptionalInt getSortOrdinal() - { - return sortOrdinal; - } - - public boolean isTemporal() - { - return temporal; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("table", table) - .add("columnId", columnId) - .add("columnName", columnName) - .add("dataType", dataType) - .toString(); - } - - public ColumnMetadata toColumnMetadata() - { - return new ColumnMetadata(columnName, dataType); - } - - public ColumnInfo toColumnInfo() - { - return new ColumnInfo(columnId, dataType); - } - - public static class Mapper - implements RowMapper - { - private final TypeManager typeManager; - - @Inject - public Mapper(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public TableColumn map(ResultSet r, StatementContext ctx) - throws SQLException - { - SchemaTableName table = new SchemaTableName( - r.getString("schema_name"), - r.getString("table_name")); - - String typeName = r.getString("data_type"); - Type type = typeManager.getType(TypeId.of(typeName)); - - return new TableColumn( - table, - r.getString("column_name"), - type, - r.getLong("column_id"), - r.getInt("ordinal_position"), - getOptionalInt(r, "bucket_ordinal_position"), - getOptionalInt(r, "sort_ordinal_position"), - r.getBoolean("temporal")); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadata.java deleted file mode 100644 index a99242b1f728..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadata.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public final class TableMetadata -{ - private final long tableId; - private final List columns; - private final List sortColumnIds; - - public TableMetadata(long tableId, List columns, List sortColumnIds) - { - this.tableId = tableId; - this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); - this.sortColumnIds = ImmutableList.copyOf(requireNonNull(sortColumnIds, "sortColumnIds is null")); - } - - public long getTableId() - { - return tableId; - } - - public List getColumns() - { - return columns; - } - - public List getSortColumnIds() - { - return sortColumnIds; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TableMetadata that = (TableMetadata) o; - return tableId == that.tableId && - Objects.equals(columns, that.columns) && - Objects.equals(sortColumnIds, that.sortColumnIds); - } - - @Override - public int hashCode() - { - return Objects.hash(tableId, columns, sortColumnIds); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("tableId", tableId) - .add("columns", columns) - .add("sortColumnIds", sortColumnIds) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadataRow.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadataRow.java deleted file mode 100644 index 29469caeaf86..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadataRow.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.core.mapper.reflect.ColumnName; - -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static java.util.Objects.requireNonNull; - -public class TableMetadataRow -{ - private final long tableId; - private final String schemaName; - private final String tableName; - private final OptionalLong temporalColumnId; - private final Optional distributionName; - private final OptionalInt bucketCount; - private final boolean organized; - - public TableMetadataRow( - long tableId, - String schemaName, - String tableName, - OptionalLong temporalColumnId, - Optional distributionName, - OptionalInt bucketCount, - @ColumnName("organization_enabled") boolean organized) - { - this.tableId = tableId; - this.schemaName = requireNonNull(schemaName, "schemaName is null"); - this.tableName = requireNonNull(tableName, "tableName is null"); - this.temporalColumnId = requireNonNull(temporalColumnId, "temporalColumnId is null"); - this.distributionName = requireNonNull(distributionName, "distributionName is null"); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.organized = organized; - } - - public long getTableId() - { - return tableId; - } - - public String getSchemaName() - { - return schemaName; - } - - public String getTableName() - { - return tableName; - } - - public OptionalLong getTemporalColumnId() - { - return temporalColumnId; - } - - public Optional getDistributionName() - { - return distributionName; - } - - public OptionalInt getBucketCount() - { - return bucketCount; - } - - public boolean isOrganized() - { - return organized; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableStatsRow.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableStatsRow.java deleted file mode 100644 index 1d295dc185a4..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableStatsRow.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import static java.util.Objects.requireNonNull; - -public class TableStatsRow -{ - private final String schemaName; - private final String tableName; - private final long createTime; - private final long updateTime; - private final long tableVersion; - private final long shardCount; - private final long rowCount; - private final long compressedSize; - private final long uncompressedSize; - - public TableStatsRow( - String schemaName, - String tableName, - long createTime, - long updateTime, - long tableVersion, - long shardCount, - long rowCount, - long compressedSize, - long uncompressedSize) - { - this.schemaName = requireNonNull(schemaName, "schemaName is null"); - this.tableName = requireNonNull(tableName, "tableName is null"); - this.createTime = createTime; - this.updateTime = updateTime; - this.tableVersion = tableVersion; - this.shardCount = shardCount; - this.rowCount = rowCount; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - } - - public String getSchemaName() - { - return schemaName; - } - - public String getTableName() - { - return tableName; - } - - public long getCreateTime() - { - return createTime; - } - - public long getUpdateTime() - { - return updateTime; - } - - public long getTableVersion() - { - return tableVersion; - } - - public long getShardCount() - { - return shardCount; - } - - public long getRowCount() - { - return rowCount; - } - - public long getCompressedSize() - { - return compressedSize; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ViewResult.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ViewResult.java deleted file mode 100644 index 76579db13967..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ViewResult.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.spi.connector.SchemaTableName; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import static java.util.Objects.requireNonNull; - -public class ViewResult -{ - private final SchemaTableName name; - private final String data; - - public ViewResult(SchemaTableName name, String data) - { - this.name = requireNonNull(name, "name is null"); - this.data = requireNonNull(data, "data is null"); - } - - public SchemaTableName getName() - { - return name; - } - - public String getData() - { - return data; - } - - public static class Mapper - implements RowMapper - { - @Override - public ViewResult map(ResultSet r, StatementContext ctx) - throws SQLException - { - SchemaTableName name = new SchemaTableName( - r.getString("schema_name"), - r.getString("table_name")); - return new ViewResult(name, r.getString("data")); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurity.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurity.java deleted file mode 100644 index bad1aacc2aaf..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurity.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -public enum RaptorSecurity -{ - ALLOW_ALL, - FILE, - READ_ONLY, -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityConfig.java deleted file mode 100644 index f62552835047..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import io.airlift.configuration.Config; -import jakarta.validation.constraints.NotNull; - -public class RaptorSecurityConfig -{ - private RaptorSecurity securitySystem = RaptorSecurity.ALLOW_ALL; - - @NotNull - public RaptorSecurity getSecuritySystem() - { - return securitySystem; - } - - @Config("raptor.security") - public RaptorSecurityConfig setSecuritySystem(RaptorSecurity securitySystem) - { - this.securitySystem = securitySystem; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityModule.java deleted file mode 100644 index b22d813660aa..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityModule.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import com.google.inject.Binder; -import com.google.inject.Module; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.base.security.ConnectorAccessControlModule; -import io.trino.plugin.base.security.FileBasedAccessControlModule; -import io.trino.plugin.base.security.ReadOnlySecurityModule; - -import static io.airlift.configuration.ConditionalModule.conditionalModule; -import static io.trino.plugin.raptor.legacy.security.RaptorSecurity.FILE; -import static io.trino.plugin.raptor.legacy.security.RaptorSecurity.READ_ONLY; - -public class RaptorSecurityModule - extends AbstractConfigurationAwareModule -{ - @Override - protected void setup(Binder binder) - { - install(new ConnectorAccessControlModule()); - bindSecurityModule(READ_ONLY, new ReadOnlySecurityModule()); - bindSecurityModule(FILE, new FileBasedAccessControlModule()); - } - - private void bindSecurityModule(RaptorSecurity raptorSecurity, Module module) - { - install(conditionalModule( - RaptorSecurityConfig.class, - security -> raptorSecurity.equals(security.getSecuritySystem()), - module)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BackupStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BackupStats.java deleted file mode 100644 index 8d6c25f19621..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BackupStats.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.errorprone.annotations.ThreadSafe; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import static io.trino.plugin.raptor.legacy.storage.ShardRecoveryManager.dataRate; - -@ThreadSafe -public class BackupStats -{ - private final DistributionStat copyToBackupBytesPerSecond = new DistributionStat(); - private final DistributionStat copyToBackupShardSizeBytes = new DistributionStat(); - private final DistributionStat copyToBackupTimeInMilliSeconds = new DistributionStat(); - private final DistributionStat queuedTimeMilliSeconds = new DistributionStat(); - - private final CounterStat backupSuccess = new CounterStat(); - private final CounterStat backupFailure = new CounterStat(); - private final CounterStat backupCorruption = new CounterStat(); - - public void addCopyShardDataRate(DataSize size, Duration duration) - { - DataSize rate = dataRate(size, duration).succinct(); - copyToBackupBytesPerSecond.add(rate.toBytes()); - copyToBackupShardSizeBytes.add(size.toBytes()); - copyToBackupTimeInMilliSeconds.add(duration.toMillis()); - } - - public void addQueuedTime(Duration queuedTime) - { - queuedTimeMilliSeconds.add(queuedTime.toMillis()); - } - - public void incrementBackupSuccess() - { - backupSuccess.update(1); - } - - public void incrementBackupFailure() - { - backupFailure.update(1); - } - - public void incrementBackupCorruption() - { - backupCorruption.update(1); - } - - @Managed - @Nested - public DistributionStat getCopyToBackupBytesPerSecond() - { - return copyToBackupBytesPerSecond; - } - - @Managed - @Nested - public DistributionStat getCopyToBackupShardSizeBytes() - { - return copyToBackupShardSizeBytes; - } - - @Managed - @Nested - public DistributionStat getCopyToBackupTimeInMilliSeconds() - { - return copyToBackupTimeInMilliSeconds; - } - - @Managed - @Nested - public DistributionStat getQueuedTimeMilliSeconds() - { - return queuedTimeMilliSeconds; - } - - @Managed - @Nested - public CounterStat getBackupSuccess() - { - return backupSuccess; - } - - @Managed - @Nested - public CounterStat getBackupFailure() - { - return backupFailure; - } - - @Managed - @Nested - public CounterStat getBackupCorruption() - { - return backupCorruption; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancer.java deleted file mode 100644 index 2193e4759b56..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancer.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.VerifyException; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.HashMultiset; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multiset; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.backup.BackupService; -import io.trino.plugin.raptor.legacy.metadata.BucketNode; -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.catalog.CatalogName; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; - -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static java.util.Comparator.comparingInt; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - -/** - * Service to balance buckets across active Raptor storage nodes in a cluster. - *

The objectives of this service are: - *

    - *
  1. For a given distribution, each node should be allocated the same number of buckets - * for a distribution. This enhances parallelism, and therefore query performance. - *
  2. The total disk utilization across the cluster should be balanced. This ensures that total - * cluster storage capacity is maximized. Simply allocating the same number of buckets - * to every node may not achieve this, as bucket sizes may vary dramatically across distributions. - *
- *

This prioritizes query performance over total cluster storage capacity, and therefore may - * produce a cluster state that is imbalanced in terms of disk utilization. - */ -public class BucketBalancer -{ - private static final Logger log = Logger.get(BucketBalancer.class); - - private final NodeSupplier nodeSupplier; - private final ShardManager shardManager; - private final boolean enabled; - private final Duration interval; - private final boolean backupAvailable; - private final boolean coordinator; - private final ScheduledExecutorService executor; - - private final AtomicBoolean started = new AtomicBoolean(); - - private final CounterStat bucketsBalanced = new CounterStat(); - private final CounterStat jobErrors = new CounterStat(); - - @Inject - public BucketBalancer( - NodeManager nodeManager, - NodeSupplier nodeSupplier, - ShardManager shardManager, - BucketBalancerConfig config, - BackupService backupService, - CatalogName catalogName) - { - this(nodeSupplier, - shardManager, - config.isBalancerEnabled(), - config.getBalancerInterval(), - backupService.isBackupAvailable(), - nodeManager.getCurrentNode().isCoordinator(), - catalogName.toString()); - } - - public BucketBalancer( - NodeSupplier nodeSupplier, - ShardManager shardManager, - boolean enabled, - Duration interval, - boolean backupAvailable, - boolean coordinator, - String catalogName) - { - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.enabled = enabled; - this.interval = requireNonNull(interval, "interval is null"); - this.backupAvailable = backupAvailable; - this.coordinator = coordinator; - this.executor = newSingleThreadScheduledExecutor(daemonThreadsNamed("bucket-balancer-" + catalogName)); - } - - @PostConstruct - public void start() - { - if (enabled && backupAvailable && coordinator && !started.getAndSet(true)) { - executor.scheduleWithFixedDelay(this::runBalanceJob, interval.toMillis(), interval.toMillis(), MILLISECONDS); - } - } - - @PreDestroy - public void shutdown() - { - executor.shutdownNow(); - } - - @Managed - @Nested - public CounterStat getBucketsBalanced() - { - return bucketsBalanced; - } - - @Managed - @Nested - public CounterStat getJobErrors() - { - return jobErrors; - } - - @Managed - public void startBalanceJob() - { - executor.submit(this::runBalanceJob); - } - - private void runBalanceJob() - { - try { - balance(); - } - catch (Throwable t) { - log.error(t, "Error balancing buckets"); - jobErrors.update(1); - } - } - - @VisibleForTesting - synchronized int balance() - { - log.info("Bucket balancer started. Computing assignments..."); - Multimap sourceToAssignmentChanges = computeAssignmentChanges(fetchClusterState()); - - log.info("Moving buckets..."); - int moves = updateAssignments(sourceToAssignmentChanges); - - log.info("Bucket balancing finished. Moved %s buckets.", moves); - return moves; - } - - private static Multimap computeAssignmentChanges(ClusterState clusterState) - { - Multimap sourceToAllocationChanges = HashMultimap.create(); - - Map allocationBytes = new HashMap<>(clusterState.getAssignedBytes()); - Set activeNodes = clusterState.getActiveNodes(); - - for (Distribution distribution : clusterState.getDistributionAssignments().keySet()) { - // number of buckets in this distribution assigned to a node - Multiset allocationCounts = HashMultiset.create(); - Collection distributionAssignments = clusterState.getDistributionAssignments().get(distribution); - distributionAssignments.stream() - .map(BucketAssignment::getNodeIdentifier) - .forEach(allocationCounts::add); - - int currentMin = allocationBytes.keySet().stream() - .mapToInt(allocationCounts::count) - .min() - .getAsInt(); - int currentMax = allocationBytes.keySet().stream() - .mapToInt(allocationCounts::count) - .max() - .getAsInt(); - - int numBuckets = distributionAssignments.size(); - int targetMin = (int) Math.floor((numBuckets * 1.0) / clusterState.getActiveNodes().size()); - int targetMax = (int) Math.ceil((numBuckets * 1.0) / clusterState.getActiveNodes().size()); - - log.info("Distribution %s: Current bucket skew: min %s, max %s. Target bucket skew: min %s, max %s", distribution.getId(), currentMin, currentMax, targetMin, targetMax); - - for (String source : ImmutableSet.copyOf(allocationCounts)) { - List existingAssignments = distributionAssignments.stream() - .filter(assignment -> assignment.getNodeIdentifier().equals(source)) - .collect(toList()); - - for (BucketAssignment existingAssignment : existingAssignments) { - if (activeNodes.contains(source) && allocationCounts.count(source) <= targetMin) { - break; - } - - // identify nodes with bucket counts lower than the computed target, and greedily select from this set based on projected disk utilization. - // greediness means that this may produce decidedly non-optimal results if one looks at the global distribution of buckets->nodes. - // also, this assumes that nodes in a cluster have identical storage capacity - String target = activeNodes.stream() - .filter(candidate -> !candidate.equals(source) && allocationCounts.count(candidate) < targetMax) - .sorted(comparingInt(allocationCounts::count)) - .min(Comparator.comparingDouble(allocationBytes::get)) - .orElseThrow(() -> new VerifyException("unable to find target for rebalancing")); - - long bucketSize = clusterState.getDistributionBucketSize().get(distribution); - - // only move bucket if it reduces imbalance - if (activeNodes.contains(source) && (allocationCounts.count(source) == targetMax && allocationCounts.count(target) == targetMin)) { - break; - } - - allocationCounts.remove(source); - allocationCounts.add(target); - allocationBytes.compute(source, (k, v) -> v - bucketSize); - allocationBytes.compute(target, (k, v) -> v + bucketSize); - - sourceToAllocationChanges.put( - existingAssignment.getNodeIdentifier(), - new BucketAssignment(existingAssignment.getDistributionId(), existingAssignment.getBucketNumber(), target)); - } - } - } - - return sourceToAllocationChanges; - } - - private int updateAssignments(Multimap sourceToAllocationChanges) - { - // perform moves in decreasing order of source node total assigned buckets - List sourceNodes = sourceToAllocationChanges.asMap().entrySet().stream() - .sorted((a, b) -> Integer.compare(b.getValue().size(), a.getValue().size())) - .map(Map.Entry::getKey) - .collect(toList()); - - int moves = 0; - for (String source : sourceNodes) { - for (BucketAssignment reassignment : sourceToAllocationChanges.get(source)) { - // todo: rate-limit new assignments - shardManager.updateBucketAssignment(reassignment.getDistributionId(), reassignment.getBucketNumber(), reassignment.getNodeIdentifier()); - bucketsBalanced.update(1); - moves++; - log.info("Distribution %s: Moved bucket %s from %s to %s", - reassignment.getDistributionId(), - reassignment.getBucketNumber(), - source, - reassignment.getNodeIdentifier()); - } - } - - return moves; - } - - @VisibleForTesting - ClusterState fetchClusterState() - { - Set activeNodes = nodeSupplier.getWorkerNodes().stream() - .map(Node::getNodeIdentifier) - .collect(toSet()); - - Map assignedNodeSize = new HashMap<>(activeNodes.stream().collect(toMap(node -> node, node -> 0L))); - ImmutableMultimap.Builder distributionAssignments = ImmutableMultimap.builder(); - ImmutableMap.Builder distributionBucketSize = ImmutableMap.builder(); - - for (Distribution distribution : shardManager.getDistributions()) { - long distributionSize = shardManager.getDistributionSizeInBytes(distribution.getId()); - long bucketSize = (long) (1.0 * distributionSize) / distribution.getBucketCount(); - distributionBucketSize.put(distribution, bucketSize); - - for (BucketNode bucketNode : shardManager.getBucketNodes(distribution.getId())) { - String node = bucketNode.getNodeIdentifier(); - distributionAssignments.put(distribution, new BucketAssignment(distribution.getId(), bucketNode.getBucketNumber(), node)); - assignedNodeSize.merge(node, bucketSize, Math::addExact); - } - } - - return new ClusterState(activeNodes, assignedNodeSize, distributionAssignments.build(), distributionBucketSize.buildOrThrow()); - } - - @VisibleForTesting - static class ClusterState - { - private final Set activeNodes; - private final Map assignedBytes; - private final Multimap distributionAssignments; - private final Map distributionBucketSize; - - public ClusterState( - Set activeNodes, - Map assignedBytes, - Multimap distributionAssignments, - Map distributionBucketSize) - { - this.activeNodes = ImmutableSet.copyOf(requireNonNull(activeNodes, "activeNodes is null")); - this.assignedBytes = ImmutableMap.copyOf(requireNonNull(assignedBytes, "assignedBytes is null")); - this.distributionAssignments = ImmutableMultimap.copyOf(requireNonNull(distributionAssignments, "distributionAssignments is null")); - this.distributionBucketSize = ImmutableMap.copyOf(requireNonNull(distributionBucketSize, "distributionBucketSize is null")); - } - - public Set getActiveNodes() - { - return activeNodes; - } - - public Map getAssignedBytes() - { - return assignedBytes; - } - - public Multimap getDistributionAssignments() - { - return distributionAssignments; - } - - public Map getDistributionBucketSize() - { - return distributionBucketSize; - } - } - - @VisibleForTesting - static class BucketAssignment - { - private final long distributionId; - private final int bucketNumber; - private final String nodeIdentifier; - - public BucketAssignment(long distributionId, int bucketNumber, String nodeIdentifier) - { - this.distributionId = distributionId; - this.bucketNumber = bucketNumber; - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - } - - public long getDistributionId() - { - return distributionId; - } - - public int getBucketNumber() - { - return bucketNumber; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancerConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancerConfig.java deleted file mode 100644 index 268d0ba1d237..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancerConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; - -import java.util.concurrent.TimeUnit; - -public class BucketBalancerConfig -{ - private boolean balancerEnabled = true; - private Duration balancerInterval = new Duration(6, TimeUnit.HOURS); - - public boolean isBalancerEnabled() - { - return balancerEnabled; - } - - @Config("storage.balancer-enabled") - public BucketBalancerConfig setBalancerEnabled(boolean balancerEnabled) - { - this.balancerEnabled = balancerEnabled; - return this; - } - - public Duration getBalancerInterval() - { - return balancerInterval; - } - - @Config("storage.balancer-interval") - @ConfigDescription("How often to run the global bucket balancer") - public BucketBalancerConfig setBalancerInterval(Duration balancerInterval) - { - this.balancerInterval = balancerInterval; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ColumnIndexStatsUtils.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ColumnIndexStatsUtils.java deleted file mode 100644 index 57a643b112ea..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ColumnIndexStatsUtils.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.sql.JDBCType; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; - -public final class ColumnIndexStatsUtils -{ - private ColumnIndexStatsUtils() {} - - public static JDBCType jdbcType(Type type) - { - if (type.equals(BOOLEAN)) { - return JDBCType.BOOLEAN; - } - if (type.equals(BIGINT) || type.equals(TIMESTAMP_MILLIS)) { - return JDBCType.BIGINT; - } - if (type.equals(INTEGER)) { - return JDBCType.INTEGER; - } - if (type.equals(DOUBLE)) { - return JDBCType.DOUBLE; - } - if (type.equals(DATE)) { - return JDBCType.INTEGER; - } - if (type instanceof VarcharType) { - return JDBCType.VARBINARY; - } - return null; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/FileStorageService.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/FileStorageService.java deleted file mode 100644 index 29e2fb6f0a1a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/FileStorageService.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.trino.spi.TrinoException; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.regex.Pattern; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.requireNonNull; - -public class FileStorageService - implements StorageService -{ - private static final Logger log = Logger.get(FileStorageService.class); - - private static final Pattern HEX_DIRECTORY = Pattern.compile("[0-9a-f]{2}"); - private static final String FILE_EXTENSION = ".orc"; - - private final File baseStorageDir; - private final File baseStagingDir; - private final File baseQuarantineDir; - - @Inject - public FileStorageService(StorageManagerConfig config) - { - this(config.getDataDirectory()); - } - - public FileStorageService(File dataDirectory) - { - File baseDataDir = requireNonNull(dataDirectory, "dataDirectory is null"); - this.baseStorageDir = new File(baseDataDir, "storage"); - this.baseStagingDir = new File(baseDataDir, "staging"); - this.baseQuarantineDir = new File(baseDataDir, "quarantine"); - } - - @Override - @PostConstruct - public void start() - { - deleteStagingFilesAsync(); - createParents(new File(baseStagingDir, ".")); - createParents(new File(baseStorageDir, ".")); - createParents(new File(baseQuarantineDir, ".")); - } - - @Override - public long getAvailableBytes() - { - return baseStorageDir.getUsableSpace(); - } - - @PreDestroy - public void stop() - throws IOException - { - deleteDirectory(baseStagingDir); - } - - @Override - public File getStorageFile(UUID shardUuid) - { - return getFileSystemPath(baseStorageDir, shardUuid); - } - - @Override - public File getStagingFile(UUID shardUuid) - { - String name = getFileSystemPath(new File("/"), shardUuid).getName(); - return new File(baseStagingDir, name); - } - - @Override - public File getQuarantineFile(UUID shardUuid) - { - String name = getFileSystemPath(new File("/"), shardUuid).getName(); - return new File(baseQuarantineDir, name); - } - - @Override - public Set getStorageShards() - { - ImmutableSet.Builder shards = ImmutableSet.builder(); - for (File level1 : listFiles(baseStorageDir, FileStorageService::isHexDirectory)) { - for (File level2 : listFiles(level1, FileStorageService::isHexDirectory)) { - for (File file : listFiles(level2, path -> true)) { - if (file.isFile()) { - uuidFromFileName(file.getName()).ifPresent(shards::add); - } - } - } - } - return shards.build(); - } - - @Override - public void createParents(File file) - { - File dir = file.getParentFile(); - if (!dir.mkdirs() && !dir.isDirectory()) { - throw new TrinoException(RAPTOR_ERROR, "Failed creating directories: " + dir); - } - } - - /** - * Generate a file system path for a shard UUID. - * This creates a three level deep directory structure where the first - * two levels each contain two hex digits (lowercase) of the UUID - * and the final level contains the full UUID. Example: - * 

-     * UUID: 701e1a79-74f7-4f56-b438-b41e8e7d019d
-     * Path: /base/70/1e/701e1a79-74f7-4f56-b438-b41e8e7d019d.orc
-     *
- * This ensures that files are spread out evenly through the tree - * while a path can still be easily navigated by a human being. - */ - public static File getFileSystemPath(File base, UUID shardUuid) - { - String uuid = shardUuid.toString().toLowerCase(ENGLISH); - return base.toPath() - .resolve(uuid.substring(0, 2)) - .resolve(uuid.substring(2, 4)) - .resolve(uuid + FILE_EXTENSION) - .toFile(); - } - - private void deleteStagingFilesAsync() - { - List files = listFiles(baseStagingDir, null); - if (!files.isEmpty()) { - new Thread(() -> { - for (File file : files) { - try { - Files.deleteIfExists(file.toPath()); - } - catch (IOException e) { - log.warn(e, "Failed to delete file: %s", file.getAbsolutePath()); - } - } - }, "background-staging-delete").start(); - } - } - - private static void deleteDirectory(File dir) - throws IOException - { - if (!dir.exists()) { - return; - } - File[] files = dir.listFiles(); - if (files == null) { - throw new IOException("Failed to list directory: " + dir); - } - for (File file : files) { - Files.deleteIfExists(file.toPath()); - } - Files.deleteIfExists(dir.toPath()); - } - - private static List listFiles(File dir, FileFilter filter) - { - File[] files = dir.listFiles(filter); - if (files == null) { - return ImmutableList.of(); - } - return ImmutableList.copyOf(files); - } - - private static boolean isHexDirectory(File file) - { - return file.isDirectory() && HEX_DIRECTORY.matcher(file.getName()).matches(); - } - - private static Optional uuidFromFileName(String name) - { - if (name.endsWith(FILE_EXTENSION)) { - name = name.substring(0, name.length() - FILE_EXTENSION.length()); - return uuidFromString(name); - } - return Optional.empty(); - } - - private static Optional uuidFromString(String value) - { - try { - return Optional.of(UUID.fromString(value)); - } - catch (IllegalArgumentException e) { - return Optional.empty(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileMetadata.java deleted file mode 100644 index b96994e9ca7c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileMetadata.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableMap; -import io.trino.spi.type.TypeId; - -import java.util.Map; -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; - -public class OrcFileMetadata -{ - static final String KEY = "metadata"; - - private final Map columnTypes; - - @JsonCreator - public OrcFileMetadata(@JsonProperty("columnTypes") Map columnTypes) - { - this.columnTypes = ImmutableMap.copyOf(columnTypes); - } - - @JsonProperty - public Map getColumnTypes() - { - return columnTypes; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OrcFileMetadata that = (OrcFileMetadata) o; - return Objects.equals(columnTypes, that.columnTypes); - } - - @Override - public int hashCode() - { - return Objects.hash(columnTypes); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("columnTypes", columnTypes) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileRewriter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileRewriter.java deleted file mode 100644 index 3d154a6b6f0b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileRewriter.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.Maps; -import io.airlift.log.Logger; -import io.airlift.slice.Slice; -import io.trino.orc.FileOrcDataSource; -import io.trino.orc.OrcPredicate; -import io.trino.orc.OrcReader; -import io.trino.orc.OrcReaderOptions; -import io.trino.orc.OrcRecordReader; -import io.trino.orc.OrcWriter; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.DictionaryBlock; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import org.joda.time.DateTimeZone; - -import java.io.File; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.util.BitSet; -import java.util.List; -import java.util.Map; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static io.trino.orc.OrcReader.INITIAL_BATCH_SIZE; -import static io.trino.orc.OrcReader.createOrcReader; -import static io.trino.plugin.raptor.legacy.storage.OrcFileWriter.createOrcFileWriter; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.getColumnInfo; -import static java.lang.Math.toIntExact; - -public final class OrcFileRewriter -{ - private static final Logger log = Logger.get(OrcFileRewriter.class); - - private OrcFileRewriter() {} - - public static OrcFileInfo rewrite(TypeManager typeManager, File input, File output, BitSet rowsToDelete) - throws IOException - { - OrcReaderOptions options = new OrcReaderOptions(); - OrcReader reader = createOrcReader(new FileOrcDataSource(input, options), options) - .orElseThrow(() -> new IOException("File is empty: " + input)); - return rewrite(typeManager, reader, output, rowsToDelete); - } - - public static OrcFileInfo rewrite(TypeManager typeManager, OrcReader reader, File output, BitSet rowsToDelete) - throws IOException - { - long start = System.nanoTime(); - - List columnInfo = getColumnInfo(typeManager, reader); - - List columnNames = columnInfo.stream() - .map(info -> String.valueOf(info.getColumnId())) - .collect(toImmutableList()); - - List columnTypes = columnInfo.stream() - .map(ColumnInfo::getType) - .collect(toImmutableList()); - - OrcRecordReader recordReader = reader.createRecordReader( - reader.getRootColumn().getNestedColumns(), - columnTypes, - OrcPredicate.TRUE, - DateTimeZone.UTC, - newSimpleAggregatedMemoryContext(), - INITIAL_BATCH_SIZE, - RaptorPageSource::handleException); - - long fileRowCount = recordReader.getFileRowCount(); - if (fileRowCount < rowsToDelete.length()) { - throw new IOException("File has fewer rows than deletion vector"); - } - int deleteRowCount = rowsToDelete.cardinality(); - if (fileRowCount == deleteRowCount) { - return new OrcFileInfo(0, 0); - } - if (fileRowCount >= Integer.MAX_VALUE) { - throw new IOException("File has too many rows"); - } - - Map metadata = Maps.transformValues(reader.getFooter().getUserMetadata(), Slice::toStringUtf8); - - OrcFileInfo fileInfo; - try (OrcWriter writer = createOrcFileWriter(output, columnNames, columnTypes, reader.getFooter().getTypes(), metadata)) { - fileInfo = rewrite(recordReader, writer, rowsToDelete); - } - log.debug("Rewrote file in %s (input rows: %s, output rows: %s)", nanosSince(start), fileRowCount, fileRowCount - deleteRowCount); - return fileInfo; - } - - private static OrcFileInfo rewrite(OrcRecordReader reader, OrcWriter writer, BitSet rowsToDelete) - throws IOException - { - long rowCount = 0; - long uncompressedSize = 0; - - while (true) { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedIOException(); - } - - Page page = reader.nextPage(); - if (page == null) { - break; - } - - int start = toIntExact(reader.getFilePosition()); - page = maskedPage(page, rowsToDelete, start); - writer.write(page); - - rowCount += page.getPositionCount(); - uncompressedSize += page.getLogicalSizeInBytes(); - } - - return new OrcFileInfo(rowCount, uncompressedSize); - } - - private static Page maskedPage(Page page, BitSet rowsToDelete, int start) - { - int end = start + page.getPositionCount(); - if (rowsToDelete.nextSetBit(start) >= end) { - return page; - } - if (rowsToDelete.nextClearBit(start) >= end) { - return page.copyPositions(new int[0], 0, 0); - } - - int[] ids = new int[page.getPositionCount()]; - int size = 0; - for (int i = 0; i < ids.length; i++) { - if (!rowsToDelete.get(start + i)) { - ids[size] = i; - size++; - } - } - - Block[] maskedBlocks = new Block[page.getChannelCount()]; - for (int i = 0; i < maskedBlocks.length; i++) { - maskedBlocks[i] = DictionaryBlock.create(size, page.getBlock(i), ids); - } - return new Page(maskedBlocks); - } - - public static class OrcFileInfo - { - private final long rowCount; - private final long uncompressedSize; - - public OrcFileInfo(long rowCount, long uncompressedSize) - { - this.rowCount = rowCount; - this.uncompressedSize = uncompressedSize; - } - - public long getRowCount() - { - return rowCount; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileWriter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileWriter.java deleted file mode 100644 index 142e278bc01c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileWriter.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.trino.orc.OrcDataSink; -import io.trino.orc.OrcWriteValidation.OrcWriteValidationMode; -import io.trino.orc.OrcWriter; -import io.trino.orc.OrcWriterOptions; -import io.trino.orc.OrcWriterStats; -import io.trino.orc.OutputStreamOrcDataSink; -import io.trino.orc.metadata.ColumnMetadata; -import io.trino.orc.metadata.CompressionKind; -import io.trino.orc.metadata.OrcType; -import io.trino.plugin.raptor.legacy.util.SyncingFileOutputStream; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeManager; - -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.trino.orc.metadata.OrcType.createRootOrcType; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.toOrcFileType; - -public class OrcFileWriter - implements Closeable -{ - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - private final PageBuilder pageBuilder; - private final OrcWriter orcWriter; - - private boolean closed; - private long rowCount; - private long uncompressedSize; - - public OrcFileWriter(TypeManager typeManager, List columnIds, List columnTypes, File target) - { - checkArgument(columnIds.size() == columnTypes.size(), "ids and types mismatch"); - checkArgument(isUnique(columnIds), "ids must be unique"); - - List columnNames = columnIds.stream() - .map(Objects::toString) - .collect(toImmutableList()); - - columnTypes = columnTypes.stream() - .map(type -> toOrcFileType(type, typeManager)) - .collect(toImmutableList()); - - this.pageBuilder = new PageBuilder(columnTypes); - - ColumnMetadata orcTypes = createRootOrcType(columnNames, columnTypes); - Map metadata = createFileMetadata(columnIds, columnTypes); - - this.orcWriter = createOrcFileWriter(target, columnNames, columnTypes, orcTypes, metadata); - } - - public void appendPages(List pages) - { - for (Page page : pages) { - appendPage(page); - } - } - - public void appendPages(List pages, int[] pageIndexes, int[] positionIndexes) - { - checkArgument(pageIndexes.length == positionIndexes.length, "pageIndexes and positionIndexes do not match"); - - for (int i = 0; i < pageIndexes.length; i++) { - Page page = pages.get(pageIndexes[i]); - int position = positionIndexes[i]; - - pageBuilder.declarePosition(); - for (int channel = 0; channel < page.getChannelCount(); channel++) { - Block block = page.getBlock(channel); - BlockBuilder output = pageBuilder.getBlockBuilder(channel); - pageBuilder.getType(channel).appendTo(block, position, output); - } - - if (pageBuilder.isFull()) { - appendPage(pageBuilder.build()); - pageBuilder.reset(); - } - } - - if (!pageBuilder.isEmpty()) { - appendPage(pageBuilder.build()); - pageBuilder.reset(); - } - } - - private void appendPage(Page page) - { - rowCount += page.getPositionCount(); - uncompressedSize += page.getLogicalSizeInBytes(); - - try { - orcWriter.write(page); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to write data", e); - } - } - - public long getRowCount() - { - return rowCount; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - - @Override - public void close() - { - if (closed) { - return; - } - closed = true; - - try { - orcWriter.close(); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to close writer", e); - } - } - - private static boolean isUnique(Collection items) - { - return new HashSet<>(items).size() == items.size(); - } - - private static Map createFileMetadata(List columnIds, List columnTypes) - { - ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); - for (int i = 0; i < columnIds.size(); i++) { - columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeId()); - } - OrcFileMetadata metadata = new OrcFileMetadata(columnTypesMap.buildOrThrow()); - return ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(metadata)); - } - - private static OrcDataSink createOrcDataSink(File target) - { - try { - return OutputStreamOrcDataSink.create(new SyncingFileOutputStream(target)); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to open output file: " + target, e); - } - } - - public static OrcWriter createOrcFileWriter( - File target, - List columnNames, - List types, - ColumnMetadata orcTypes, - Map metadata) - { - return new OrcWriter( - createOrcDataSink(target), - columnNames, - types, - orcTypes, - CompressionKind.SNAPPY, - new OrcWriterOptions(), - metadata, - true, - OrcWriteValidationMode.BOTH, - new OrcWriterStats()); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java deleted file mode 100644 index bbc7aba4661d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.memory.context.AggregatedMemoryContext; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcRecordReader; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.block.RowBlock; -import io.trino.spi.block.RunLengthEncodedBlock; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.type.Type; -import io.trino.spi.type.UuidType; - -import java.io.IOException; -import java.util.List; -import java.util.OptionalInt; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.base.util.Closables.closeAllSuppress; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.UuidType.javaUuidToTrinoUuid; -import static java.util.Objects.requireNonNull; - -public class RaptorPageSource - implements ConnectorPageSource -{ - private final OrcRecordReader recordReader; - private final List columnAdaptations; - private final OrcDataSource orcDataSource; - - private final AggregatedMemoryContext memoryContext; - - private boolean closed; - - public RaptorPageSource( - OrcRecordReader recordReader, - List columnAdaptations, - OrcDataSource orcDataSource, - AggregatedMemoryContext memoryContext) - { - this.recordReader = requireNonNull(recordReader, "recordReader is null"); - this.columnAdaptations = ImmutableList.copyOf(requireNonNull(columnAdaptations, "columnAdaptations is null")); - this.orcDataSource = requireNonNull(orcDataSource, "orcDataSource is null"); - - this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); - } - - @Override - public long getCompletedBytes() - { - return orcDataSource.getReadBytes(); - } - - @Override - public long getReadTimeNanos() - { - return orcDataSource.getReadTimeNanos(); - } - - @Override - public boolean isFinished() - { - return closed; - } - - @Override - public Page getNextPage() - { - Page page; - try { - page = recordReader.nextPage(); - } - catch (IOException | RuntimeException e) { - closeAllSuppress(e, this); - throw handleException(e); - } - - if (page == null) { - close(); - return null; - } - - long filePosition = recordReader.getFilePosition(); - Block[] blocks = new Block[columnAdaptations.size()]; - for (int i = 0; i < columnAdaptations.size(); i++) { - blocks[i] = columnAdaptations.get(i).block(page, filePosition); - } - return new Page(page.getPositionCount(), blocks); - } - - public static TrinoException handleException(Exception exception) - { - if (exception instanceof TrinoException) { - return (TrinoException) exception; - } - throw new TrinoException(RAPTOR_ERROR, exception); - } - - @Override - public void close() - { - if (closed) { - return; - } - closed = true; - - try { - recordReader.close(); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, e); - } - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("columns", columnAdaptations) - .toString(); - } - - @Override - public long getMemoryUsage() - { - return memoryContext.getBytes(); - } - - public interface ColumnAdaptation - { - Block block(Page sourcePage, long filePosition); - - static ColumnAdaptation nullColumn(Type type) - { - return new NullColumn(type); - } - - static ColumnAdaptation shardUuidColumn(UUID shardUuid) - { - return new ShardUuidAdaptation(shardUuid); - } - - static ColumnAdaptation bucketNumberColumn(OptionalInt bucketNumber) - { - if (bucketNumber.isEmpty()) { - return nullColumn(INTEGER); - } - return new BucketNumberColumn(bucketNumber.getAsInt()); - } - - static ColumnAdaptation rowIdColumn() - { - return RowIdColumn.INSTANCE; - } - - static ColumnAdaptation mergeRowIdColumn(OptionalInt bucketNumber, UUID shardUuid) - { - return new MergeRowIdColumn(bucketNumber, shardUuid); - } - - static ColumnAdaptation sourceColumn(int index) - { - return new SourceColumn(index); - } - } - - private static class ShardUuidAdaptation - implements ColumnAdaptation - { - private final Block shardUuidBlock; - - public ShardUuidAdaptation(UUID shardUuid) - { - Slice slice = utf8Slice(shardUuid.toString()); - BlockBuilder blockBuilder = SHARD_UUID_COLUMN_TYPE.createBlockBuilder(null, 1, slice.length()); - SHARD_UUID_COLUMN_TYPE.writeSlice(blockBuilder, slice); - this.shardUuidBlock = blockBuilder.build(); - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - return RunLengthEncodedBlock.create(shardUuidBlock, sourcePage.getPositionCount()); - } - - @Override - public String toString() - { - return toStringHelper(this) - .toString(); - } - } - - private static class RowIdColumn - implements ColumnAdaptation - { - public static final RowIdColumn INSTANCE = new RowIdColumn(); - - @Override - public Block block(Page sourcePage, long filePosition) - { - int count = sourcePage.getPositionCount(); - BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(count); - for (int i = 0; i < count; i++) { - BIGINT.writeLong(builder, filePosition + i); - } - return builder.build(); - } - - @Override - public String toString() - { - return toStringHelper(this) - .toString(); - } - } - - private static class MergeRowIdColumn - implements ColumnAdaptation - { - private final Block bucketNumberValue; - private final Block shardUuidValue; - - public MergeRowIdColumn(OptionalInt bucketNumber, UUID shardUuid) - { - BlockBuilder blockBuilder = INTEGER.createFixedSizeBlockBuilder(1); - bucketNumber.ifPresentOrElse(value -> INTEGER.writeLong(blockBuilder, value), blockBuilder::appendNull); - bucketNumberValue = blockBuilder.build(); - - BlockBuilder builder = UuidType.UUID.createFixedSizeBlockBuilder(1); - UuidType.UUID.writeSlice(builder, javaUuidToTrinoUuid(shardUuid)); - shardUuidValue = builder.build(); - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - Block bucketNumberBlock = RunLengthEncodedBlock.create(bucketNumberValue, sourcePage.getPositionCount()); - Block shardUuidBlock = RunLengthEncodedBlock.create(shardUuidValue, sourcePage.getPositionCount()); - Block rowIdBlock = RowIdColumn.INSTANCE.block(sourcePage, filePosition); - return RowBlock.fromFieldBlocks( - sourcePage.getPositionCount(), - new Block[] {bucketNumberBlock, shardUuidBlock, rowIdBlock}); - } - } - - private static class NullColumn - implements ColumnAdaptation - { - private final Type type; - private final Block nullBlock; - - public NullColumn(Type type) - { - this.type = requireNonNull(type, "type is null"); - this.nullBlock = type.createBlockBuilder(null, 1, 0) - .appendNull() - .build(); - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - return RunLengthEncodedBlock.create(nullBlock, sourcePage.getPositionCount()); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("type", type) - .toString(); - } - } - - private static class BucketNumberColumn - implements ColumnAdaptation - { - private final Block bucketNumberBlock; - - public BucketNumberColumn(int bucketNumber) - { - BlockBuilder blockBuilder = INTEGER.createFixedSizeBlockBuilder(1); - INTEGER.writeLong(blockBuilder, bucketNumber); - this.bucketNumberBlock = blockBuilder.build(); - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - return RunLengthEncodedBlock.create(bucketNumberBlock, sourcePage.getPositionCount()); - } - - @Override - public String toString() - { - return toStringHelper(this) - .toString(); - } - } - - private static class SourceColumn - implements ColumnAdaptation - { - private final int index; - - public SourceColumn(int index) - { - checkArgument(index >= 0, "index is negative"); - this.index = index; - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - return sourcePage.getBlock(index); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("index", index) - .toString(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java deleted file mode 100644 index 16710298979e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java +++ /dev/null @@ -1,668 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.airlift.json.JsonCodec; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.airlift.slice.XxHash64; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.memory.context.AggregatedMemoryContext; -import io.trino.orc.FileOrcDataSource; -import io.trino.orc.OrcColumn; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcPredicate; -import io.trino.orc.OrcReader; -import io.trino.orc.OrcReaderOptions; -import io.trino.orc.OrcRecordReader; -import io.trino.orc.TupleDomainOrcPredicate; -import io.trino.orc.TupleDomainOrcPredicate.TupleDomainOrcPredicateBuilder; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.backup.BackupManager; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.ColumnStats; -import io.trino.plugin.raptor.legacy.metadata.ShardDelta; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; -import io.trino.plugin.raptor.legacy.storage.OrcFileRewriter.OrcFileInfo; -import io.trino.plugin.raptor.legacy.storage.RaptorPageSource.ColumnAdaptation; -import io.trino.spi.NodeManager; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.MapType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeSignature; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import jakarta.annotation.PreDestroy; - -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Throwables.throwIfInstanceOf; -import static com.google.common.collect.Maps.uniqueIndex; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.units.DataSize.Unit.PETABYTE; -import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static io.trino.orc.OrcReader.INITIAL_BATCH_SIZE; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isBucketNumberColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isHiddenColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isMergeRowIdColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isShardRowIdColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isShardUuidColumn; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_LOCAL_DISK_FULL; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_TIMEOUT; -import static io.trino.plugin.raptor.legacy.storage.ShardStats.computeColumnStats; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.TypeSignatureParameter.typeParameter; -import static java.lang.Math.min; -import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.completedFuture; -import static java.util.concurrent.CompletableFuture.supplyAsync; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static java.util.stream.Collectors.toList; -import static org.joda.time.DateTimeZone.UTC; - -public class RaptorStorageManager - implements StorageManager -{ - private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); - - private static final long MAX_ROWS = 1_000_000_000; - // TODO: do not limit the max size of blocks to read for now; enable the limit when the Hive connector is ready - private static final DataSize HUGE_MAX_READ_BLOCK_SIZE = DataSize.of(1, PETABYTE); - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - private final String nodeId; - private final StorageService storageService; - private final Optional backupStore; - private final OrcReaderOptions orcReaderOptions; - private final BackupManager backupManager; - private final ShardRecoveryManager recoveryManager; - private final ShardRecorder shardRecorder; - private final Duration recoveryTimeout; - private final long maxShardRows; - private final DataSize maxShardSize; - private final DataSize minAvailableSpace; - private final TypeManager typeManager; - private final ExecutorService deletionExecutor; - private final ExecutorService commitExecutor; - - @Inject - public RaptorStorageManager( - NodeManager nodeManager, - StorageService storageService, - Optional backupStore, - StorageManagerConfig config, - CatalogName catalogName, - BackupManager backgroundBackupManager, - ShardRecoveryManager recoveryManager, - ShardRecorder shardRecorder, - TypeManager typeManager) - { - this(nodeManager.getCurrentNode().getNodeIdentifier(), - storageService, - backupStore, - config.toOrcReaderOptions(), - backgroundBackupManager, - recoveryManager, - shardRecorder, - typeManager, - catalogName.toString(), - config.getDeletionThreads(), - config.getShardRecoveryTimeout(), - config.getMaxShardRows(), - config.getMaxShardSize(), - config.getMinAvailableSpace()); - } - - public RaptorStorageManager( - String nodeId, - StorageService storageService, - Optional backupStore, - OrcReaderOptions orcReaderOptions, - BackupManager backgroundBackupManager, - ShardRecoveryManager recoveryManager, - ShardRecorder shardRecorder, - TypeManager typeManager, - String connectorId, - int deletionThreads, - Duration shardRecoveryTimeout, - long maxShardRows, - DataSize maxShardSize, - DataSize minAvailableSpace) - { - this.nodeId = requireNonNull(nodeId, "nodeId is null"); - this.storageService = requireNonNull(storageService, "storageService is null"); - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.orcReaderOptions = orcReaderOptions - .withMaxReadBlockSize(HUGE_MAX_READ_BLOCK_SIZE); - - backupManager = requireNonNull(backgroundBackupManager, "backgroundBackupManager is null"); - this.recoveryManager = requireNonNull(recoveryManager, "recoveryManager is null"); - this.recoveryTimeout = requireNonNull(shardRecoveryTimeout, "shardRecoveryTimeout is null"); - - checkArgument(maxShardRows > 0, "maxShardRows must be > 0"); - this.maxShardRows = min(maxShardRows, MAX_ROWS); - this.maxShardSize = requireNonNull(maxShardSize, "maxShardSize is null"); - this.minAvailableSpace = requireNonNull(minAvailableSpace, "minAvailableSpace is null"); - this.shardRecorder = requireNonNull(shardRecorder, "shardRecorder is null"); - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - this.deletionExecutor = newFixedThreadPool(deletionThreads, daemonThreadsNamed("raptor-delete-" + connectorId + "-%s")); - this.commitExecutor = newCachedThreadPool(daemonThreadsNamed("raptor-commit-" + connectorId + "-%s")); - } - - @PreDestroy - public void shutdown() - { - deletionExecutor.shutdownNow(); - commitExecutor.shutdown(); - } - - @Override - public ConnectorPageSource getPageSource( - UUID shardUuid, - OptionalInt bucketNumber, - List columnIds, - List columnTypes, - TupleDomain effectivePredicate, - OrcReaderOptions orcReaderOptions) - { - orcReaderOptions = orcReaderOptions.withMaxReadBlockSize(HUGE_MAX_READ_BLOCK_SIZE); - OrcDataSource dataSource = openShard(shardUuid, orcReaderOptions); - - AggregatedMemoryContext memoryUsage = newSimpleAggregatedMemoryContext(); - - try { - OrcReader reader = OrcReader.createOrcReader(dataSource, orcReaderOptions) - .orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "Data file is empty for shard " + shardUuid)); - - Map indexMap = columnIdIndex(reader.getRootColumn().getNestedColumns()); - List fileReadColumn = new ArrayList<>(columnIds.size()); - List fileReadTypes = new ArrayList<>(columnIds.size()); - List columnAdaptations = new ArrayList<>(columnIds.size()); - for (int i = 0; i < columnIds.size(); i++) { - long columnId = columnIds.get(i); - - if (isHiddenColumn(columnId)) { - columnAdaptations.add(specialColumnAdaptation(columnId, shardUuid, bucketNumber)); - continue; - } - - Type type = toOrcFileType(columnTypes.get(i), typeManager); - OrcColumn fileColumn = indexMap.get(columnId); - if (fileColumn == null) { - columnAdaptations.add(ColumnAdaptation.nullColumn(type)); - } - else { - int sourceIndex = fileReadColumn.size(); - columnAdaptations.add(ColumnAdaptation.sourceColumn(sourceIndex)); - fileReadColumn.add(fileColumn); - fileReadTypes.add(type); - } - } - - OrcPredicate predicate = getPredicate(effectivePredicate, indexMap); - - OrcRecordReader recordReader = reader.createRecordReader( - fileReadColumn, - fileReadTypes, - predicate, - UTC, - memoryUsage, - INITIAL_BATCH_SIZE, - RaptorPageSource::handleException); - - return new RaptorPageSource(recordReader, columnAdaptations, dataSource, memoryUsage); - } - catch (IOException | RuntimeException e) { - closeQuietly(dataSource); - throw new TrinoException(RAPTOR_ERROR, "Failed to create page source for shard " + shardUuid, e); - } - catch (Throwable t) { - closeQuietly(dataSource); - throw t; - } - } - - private static ColumnAdaptation specialColumnAdaptation(long columnId, UUID shardUuid, OptionalInt bucketNumber) - { - if (isShardRowIdColumn(columnId)) { - return ColumnAdaptation.rowIdColumn(); - } - if (isShardUuidColumn(columnId)) { - return ColumnAdaptation.shardUuidColumn(shardUuid); - } - if (isBucketNumberColumn(columnId)) { - return ColumnAdaptation.bucketNumberColumn(bucketNumber); - } - if (isMergeRowIdColumn(columnId)) { - return ColumnAdaptation.mergeRowIdColumn(bucketNumber, shardUuid); - } - throw new TrinoException(RAPTOR_ERROR, "Invalid column ID: " + columnId); - } - - @Override - public StoragePageSink createStoragePageSink(long transactionId, OptionalInt bucketNumber, List columnIds, List columnTypes, boolean checkSpace) - { - if (checkSpace && storageService.getAvailableBytes() < minAvailableSpace.toBytes()) { - throw new TrinoException(RAPTOR_LOCAL_DISK_FULL, "Local disk is full on node " + nodeId); - } - return new RaptorStoragePageSink(transactionId, columnIds, columnTypes, bucketNumber); - } - - @Override - public ShardRewriter createShardRewriter(long transactionId, OptionalInt bucketNumber, UUID shardUuid) - { - return rowsToDelete -> { - if (rowsToDelete.isEmpty()) { - return completedFuture(ImmutableList.of()); - } - return supplyAsync(() -> rewriteShard(transactionId, bucketNumber, shardUuid, rowsToDelete), deletionExecutor); - }; - } - - private void writeShard(UUID shardUuid) - { - if (backupStore.isPresent() && !backupStore.get().shardExists(shardUuid)) { - throw new TrinoException(RAPTOR_ERROR, "Backup does not exist after write"); - } - - File stagingFile = storageService.getStagingFile(shardUuid); - File storageFile = storageService.getStorageFile(shardUuid); - - storageService.createParents(storageFile); - - try { - Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to move shard file", e); - } - } - - @VisibleForTesting - OrcDataSource openShard(UUID shardUuid, OrcReaderOptions orcReaderOptions) - { - File file = storageService.getStorageFile(shardUuid).getAbsoluteFile(); - - if (!file.exists() && backupStore.isPresent()) { - try { - Future future = recoveryManager.recoverShard(shardUuid); - future.get(recoveryTimeout.toMillis(), TimeUnit.MILLISECONDS); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - catch (ExecutionException e) { - if (e.getCause() != null) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - } - throw new TrinoException(RAPTOR_RECOVERY_ERROR, "Error recovering shard " + shardUuid, e.getCause()); - } - catch (TimeoutException e) { - throw new TrinoException(RAPTOR_RECOVERY_TIMEOUT, "Shard is being recovered from backup. Please retry in a few minutes: " + shardUuid); - } - } - - try { - return fileOrcDataSource(orcReaderOptions, file); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to open shard file: " + file, e); - } - } - - private static FileOrcDataSource fileOrcDataSource(OrcReaderOptions orcReaderOptions, File file) - throws FileNotFoundException - { - return new FileOrcDataSource(file, orcReaderOptions); - } - - private ShardInfo createShardInfo(UUID shardUuid, OptionalInt bucketNumber, File file, Set nodes, long rowCount, long uncompressedSize) - { - return new ShardInfo(shardUuid, bucketNumber, nodes, computeShardStats(file), rowCount, file.length(), uncompressedSize, xxhash64(file)); - } - - private List computeShardStats(File file) - { - try (OrcDataSource dataSource = fileOrcDataSource(orcReaderOptions, file)) { - OrcReader reader = OrcReader.createOrcReader(dataSource, orcReaderOptions) - .orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "Data file is empty: " + file)); - - ImmutableList.Builder list = ImmutableList.builder(); - for (ColumnInfo info : getColumnInfo(typeManager, reader)) { - computeColumnStats(reader, info.getColumnId(), info.getType(), typeManager).ifPresent(list::add); - } - return list.build(); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to read file: " + file, e); - } - } - - @VisibleForTesting - Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, BitSet rowsToDelete) - { - if (rowsToDelete.isEmpty()) { - return ImmutableList.of(); - } - - UUID newShardUuid = UUID.randomUUID(); - File input = storageService.getStorageFile(shardUuid); - File output = storageService.getStagingFile(newShardUuid); - - OrcFileInfo info = rewriteFile(typeManager, input, output, rowsToDelete); - long rowCount = info.getRowCount(); - - if (rowCount == 0) { - return shardDelta(shardUuid, Optional.empty()); - } - - shardRecorder.recordCreatedShard(transactionId, newShardUuid); - - // submit for backup and wait until it finishes - getFutureValue(backupManager.submit(newShardUuid, output)); - - Set nodes = ImmutableSet.of(nodeId); - long uncompressedSize = info.getUncompressedSize(); - - ShardInfo shard = createShardInfo(newShardUuid, bucketNumber, output, nodes, rowCount, uncompressedSize); - - writeShard(newShardUuid); - - return shardDelta(shardUuid, Optional.of(shard)); - } - - private static Collection shardDelta(UUID oldShardUuid, Optional shardInfo) - { - List newShards = shardInfo.map(ImmutableList::of).orElse(ImmutableList.of()); - ShardDelta delta = new ShardDelta(ImmutableList.of(oldShardUuid), newShards); - return ImmutableList.of(Slices.wrappedBuffer(SHARD_DELTA_CODEC.toJsonBytes(delta))); - } - - private static OrcFileInfo rewriteFile(TypeManager typeManager, File input, File output, BitSet rowsToDelete) - { - try { - return OrcFileRewriter.rewrite(typeManager, input, output, rowsToDelete); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to rewrite shard file: " + input, e); - } - } - - static List getColumnInfo(TypeManager typeManager, OrcReader reader) - { - return getColumnInfoFromOrcUserMetadata(typeManager, getOrcFileMetadata(reader)); - } - - static long xxhash64(File file) - { - try (InputStream in = new FileInputStream(file)) { - return XxHash64.hash(in); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to read file: " + file, e); - } - } - - private static OrcFileMetadata getOrcFileMetadata(OrcReader reader) - { - return Optional.ofNullable(reader.getFooter().getUserMetadata().get(OrcFileMetadata.KEY)) - .map(slice -> METADATA_CODEC.fromJson(slice.getBytes())) - .orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "No Raptor metadata in file")); - } - - private static List getColumnInfoFromOrcUserMetadata(TypeManager typeManager, OrcFileMetadata orcFileMetadata) - { - return orcFileMetadata.getColumnTypes().entrySet() - .stream() - .sorted(Map.Entry.comparingByKey()) - .map(entry -> new ColumnInfo(entry.getKey(), typeManager.getType(entry.getValue()))) - .collect(toList()); - } - - static Type toOrcFileType(Type raptorType, TypeManager typeManager) - { - if ((raptorType == BOOLEAN) || - (raptorType == TINYINT) || - (raptorType == SMALLINT) || - (raptorType == INTEGER) || - (raptorType == BIGINT) || - (raptorType == REAL) || - (raptorType == DOUBLE) || - (raptorType == DATE) || - (raptorType instanceof DecimalType) || - (raptorType instanceof VarcharType) || - (raptorType instanceof VarbinaryType)) { - return raptorType; - } - if (raptorType.equals(TIMESTAMP_MILLIS)) { - // TIMESTAMPS are stored as BIGINT to avoid the poor encoding in ORC - return BIGINT; - } - if (raptorType instanceof ArrayType) { - Type elementType = toOrcFileType(((ArrayType) raptorType).getElementType(), typeManager); - return new ArrayType(elementType); - } - if (raptorType instanceof MapType) { - TypeSignature keyType = toOrcFileType(((MapType) raptorType).getKeyType(), typeManager).getTypeSignature(); - TypeSignature valueType = toOrcFileType(((MapType) raptorType).getValueType(), typeManager).getTypeSignature(); - return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(typeParameter(keyType), typeParameter(valueType))); - } - throw new TrinoException(NOT_SUPPORTED, "Unsupported type: " + raptorType); - } - - private static OrcPredicate getPredicate(TupleDomain effectivePredicate, Map indexMap) - { - TupleDomainOrcPredicateBuilder predicateBuilder = TupleDomainOrcPredicate.builder(); - effectivePredicate.getDomains().get().forEach((columnHandle, value) -> { - OrcColumn fileColumn = indexMap.get(columnHandle.getColumnId()); - if (fileColumn != null) { - predicateBuilder.addColumn(fileColumn.getColumnId(), value); - } - }); - return predicateBuilder.build(); - } - - private static Map columnIdIndex(List columns) - { - return uniqueIndex(columns, column -> Long.valueOf(column.getColumnName())); - } - - private class RaptorStoragePageSink - implements StoragePageSink - { - private final long transactionId; - private final List columnIds; - private final List columnTypes; - private final OptionalInt bucketNumber; - - private final List stagingFiles = new ArrayList<>(); - private final List shards = new ArrayList<>(); - private final List> futures = new ArrayList<>(); - - private boolean committed; - private OrcFileWriter writer; - private UUID shardUuid; - - public RaptorStoragePageSink(long transactionId, List columnIds, List columnTypes, OptionalInt bucketNumber) - { - this.transactionId = transactionId; - this.columnIds = ImmutableList.copyOf(requireNonNull(columnIds, "columnIds is null")); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - } - - @Override - public void appendPages(List pages) - { - createWriterIfNecessary(); - writer.appendPages(pages); - } - - @Override - public void appendPages(List inputPages, int[] pageIndexes, int[] positionIndexes) - { - createWriterIfNecessary(); - writer.appendPages(inputPages, pageIndexes, positionIndexes); - } - - @Override - public boolean isFull() - { - if (writer == null) { - return false; - } - return (writer.getRowCount() >= maxShardRows) || (writer.getUncompressedSize() >= maxShardSize.toBytes()); - } - - @Override - public void flush() - { - if (writer != null) { - writer.close(); - - shardRecorder.recordCreatedShard(transactionId, shardUuid); - - File stagingFile = storageService.getStagingFile(shardUuid); - futures.add(backupManager.submit(shardUuid, stagingFile)); - - Set nodes = ImmutableSet.of(nodeId); - long rowCount = writer.getRowCount(); - long uncompressedSize = writer.getUncompressedSize(); - - shards.add(createShardInfo(shardUuid, bucketNumber, stagingFile, nodes, rowCount, uncompressedSize)); - - writer = null; - shardUuid = null; - } - } - - @Override - public CompletableFuture> commit() - { - checkState(!committed, "already committed"); - committed = true; - - flush(); - - return allAsList(futures).thenApplyAsync(_ -> { - for (ShardInfo shard : shards) { - writeShard(shard.getShardUuid()); - } - return ImmutableList.copyOf(shards); - }, commitExecutor); - } - - @SuppressWarnings("ResultOfMethodCallIgnored") - @Override - public void rollback() - { - try { - if (writer != null) { - writer.close(); - writer = null; - } - } - finally { - for (File file : stagingFiles) { - file.delete(); - } - - // cancel incomplete backup jobs - futures.forEach(future -> future.cancel(true)); - - // delete completed backup shards - backupStore.ifPresent(backupStore -> { - for (ShardInfo shard : shards) { - backupStore.deleteShard(shard.getShardUuid()); - } - }); - } - } - - private void createWriterIfNecessary() - { - if (writer == null) { - shardUuid = UUID.randomUUID(); - File stagingFile = storageService.getStagingFile(shardUuid); - storageService.createParents(stagingFile); - stagingFiles.add(stagingFile); - writer = new OrcFileWriter(typeManager, columnIds, columnTypes, stagingFile); - } - } - } - - private static void closeQuietly(Closeable closeable) - { - try { - closeable.close(); - } - catch (IOException _) { - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardEjector.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardEjector.java deleted file mode 100644 index e9d4c819ab34..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardEjector.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.catalog.CatalogName; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.io.File; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Queue; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Maps.filterKeys; -import static com.google.common.collect.Maps.filterValues; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static java.lang.Math.round; -import static java.util.Comparator.comparingLong; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toCollection; -import static java.util.stream.Collectors.toSet; - -public class ShardEjector -{ - private static final Logger log = Logger.get(ShardEjector.class); - - private final String currentNode; - private final NodeSupplier nodeSupplier; - private final ShardManager shardManager; - private final StorageService storageService; - private final Duration interval; - private final Optional backupStore; - private final ScheduledExecutorService executor; - - private final AtomicBoolean started = new AtomicBoolean(); - - private final CounterStat shardsEjected = new CounterStat(); - private final CounterStat jobErrors = new CounterStat(); - - @Inject - public ShardEjector( - NodeManager nodeManager, - NodeSupplier nodeSupplier, - ShardManager shardManager, - StorageService storageService, - StorageManagerConfig config, - Optional backupStore, - CatalogName catalogName) - { - this(nodeManager.getCurrentNode().getNodeIdentifier(), - nodeSupplier, - shardManager, - storageService, - config.getShardEjectorInterval(), - backupStore, - catalogName.toString()); - } - - public ShardEjector( - String currentNode, - NodeSupplier nodeSupplier, - ShardManager shardManager, - StorageService storageService, - Duration interval, - Optional backupStore, - String connectorId) - { - this.currentNode = requireNonNull(currentNode, "currentNode is null"); - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.storageService = requireNonNull(storageService, "storageService is null"); - this.interval = requireNonNull(interval, "interval is null"); - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.executor = newScheduledThreadPool(1, daemonThreadsNamed("shard-ejector-" + connectorId)); - } - - @PostConstruct - public void start() - { - if (backupStore.isEmpty()) { - return; - } - if (!started.getAndSet(true)) { - startJob(); - } - } - - @PreDestroy - public void shutdown() - { - executor.shutdownNow(); - } - - @Managed - @Nested - public CounterStat getShardsEjected() - { - return shardsEjected; - } - - @Managed - @Nested - public CounterStat getJobErrors() - { - return jobErrors; - } - - private void startJob() - { - executor.scheduleWithFixedDelay(() -> { - try { - // jitter to avoid overloading database - long interval = this.interval.roundTo(SECONDS); - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval)); - process(); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable t) { - log.error(t, "Error ejecting shards"); - jobErrors.update(1); - } - }, 0, interval.toMillis(), MILLISECONDS); - } - - @VisibleForTesting - void process() - { - checkState(backupStore.isPresent(), "backup store must be present"); - - // get the size of assigned shards for each node - Map nodes = shardManager.getNodeBytes(); - - Set activeNodes = nodeSupplier.getWorkerNodes().stream() - .map(Node::getNodeIdentifier) - .collect(toSet()); - - // only include active nodes - nodes = new HashMap<>(filterKeys(nodes, activeNodes::contains)); - - if (nodes.isEmpty()) { - return; - } - - // get current node size - if (!nodes.containsKey(currentNode)) { - return; - } - long nodeSize = nodes.get(currentNode); - - // get average node size - long averageSize = round(nodes.values().stream() - .mapToLong(Long::longValue) - .average() - .getAsDouble()); - long maxSize = round(averageSize * 1.01); - - // skip if not above max - if (nodeSize <= maxSize) { - return; - } - - // only include nodes that are below threshold - nodes = new HashMap<>(filterValues(nodes, size -> size <= averageSize)); - - // get non-bucketed node shards by size, largest to smallest - Queue queue = shardManager.getNodeShards(currentNode).stream() - .filter(shard -> shard.getBucketNumber().isEmpty()) - .sorted(comparingLong(ShardMetadata::getCompressedSize).reversed()) - // eject shards while current node is above max - .collect(toCollection(ArrayDeque::new)); - while ((nodeSize > maxSize) && !queue.isEmpty()) { - ShardMetadata shard = queue.remove(); - long shardSize = shard.getCompressedSize(); - UUID shardUuid = shard.getShardUuid(); - - // verify backup exists - if (!backupStore.get().shardExists(shardUuid)) { - log.warn("No backup for shard: %s", shardUuid); - } - - // pick target node - String target = pickTargetNode(nodes, shardSize, averageSize); - if (target == null) { - return; - } - long targetSize = nodes.get(target); - - // stats - log.info("Moving shard %s to node %s (shard: %s, node: %s, average: %s, target: %s)", - shardUuid, target, shardSize, nodeSize, averageSize, targetSize); - shardsEjected.update(1); - - // update size - nodes.put(target, targetSize + shardSize); - nodeSize -= shardSize; - - // move assignment - shardManager.replaceShardAssignment(shard.getTableId(), shardUuid, target, false); - - // delete local file - File file = storageService.getStorageFile(shardUuid); - if (file.exists() && !file.delete()) { - log.warn("Failed to delete shard file: %s", file); - } - } - } - - private static String pickTargetNode(Map nodes, long shardSize, long maxSize) - { - while (!nodes.isEmpty()) { - String node = pickCandidateNode(nodes); - if ((nodes.get(node) + shardSize) <= maxSize) { - return node; - } - nodes.remove(node); - } - return null; - } - - private static String pickCandidateNode(Map nodes) - { - checkArgument(!nodes.isEmpty()); - if (nodes.size() == 1) { - return nodes.keySet().iterator().next(); - } - - // pick two random candidates, then choose the smaller one - List candidates = new ArrayList<>(nodes.keySet()); - Collections.shuffle(candidates); - String first = candidates.get(0); - String second = candidates.get(1); - return (nodes.get(first) <= nodes.get(second)) ? first : second; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java deleted file mode 100644 index 2a416e31eecc..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.util.PrioritizedFifoExecutor; -import io.trino.spi.NodeManager; -import io.trino.spi.TrinoException; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.gaul.modernizer_maven_annotations.SuppressModernizer; -import org.weakref.jmx.Flatten; -import org.weakref.jmx.Managed; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.util.Comparator; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination; -import static io.airlift.concurrent.MoreFutures.addExceptionCallback; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.units.DataSize.succinctBytes; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_ERROR; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; -import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toSet; - -public class ShardRecoveryManager -{ - private static final Logger log = Logger.get(ShardRecoveryManager.class); - - private final StorageService storageService; - private final Optional backupStore; - private final String nodeIdentifier; - private final ShardManager shardManager; - private final Duration missingShardDiscoveryInterval; - - private final AtomicBoolean started = new AtomicBoolean(); - private final MissingShardsQueue shardQueue; - - private final ScheduledExecutorService missingShardExecutor = newScheduledThreadPool(1, daemonThreadsNamed("missing-shard-discovery")); - private final ExecutorService executorService = newCachedThreadPool(daemonThreadsNamed("shard-recovery-%s")); - private final ShardRecoveryStats stats; - - @Inject - public ShardRecoveryManager( - StorageService storageService, - Optional backupStore, - NodeManager nodeManager, - ShardManager shardManager, - StorageManagerConfig config) - { - this(storageService, - backupStore, - nodeManager, - shardManager, - config.getMissingShardDiscoveryInterval(), - config.getRecoveryThreads()); - } - - public ShardRecoveryManager( - StorageService storageService, - Optional backupStore, - NodeManager nodeManager, - ShardManager shardManager, - Duration missingShardDiscoveryInterval, - int recoveryThreads) - { - this.storageService = requireNonNull(storageService, "storageService is null"); - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.nodeIdentifier = nodeManager.getCurrentNode().getNodeIdentifier(); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.missingShardDiscoveryInterval = requireNonNull(missingShardDiscoveryInterval, "missingShardDiscoveryInterval is null"); - this.shardQueue = new MissingShardsQueue(new PrioritizedFifoExecutor<>(executorService, recoveryThreads, new MissingShardComparator())); - this.stats = new ShardRecoveryStats(); - } - - @PostConstruct - public void start() - { - if (backupStore.isEmpty()) { - return; - } - if (started.compareAndSet(false, true)) { - scheduleRecoverMissingShards(); - } - } - - @PreDestroy - public void shutdown() - { - shutdownAndAwaitTermination(missingShardExecutor, 10, SECONDS); - shutdownAndAwaitTermination(executorService, 10, SECONDS); - } - - private void scheduleRecoverMissingShards() - { - missingShardExecutor.scheduleWithFixedDelay(() -> { - try { - // jitter to avoid overloading database - long interval = missingShardDiscoveryInterval.roundTo(SECONDS); - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval)); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - enqueueMissingShards(); - }, 0, missingShardDiscoveryInterval.toMillis(), TimeUnit.MILLISECONDS); - } - - @Managed - public void recoverMissingShards() - { - missingShardExecutor.submit(this::enqueueMissingShards); - } - - private synchronized void enqueueMissingShards() - { - try { - for (ShardMetadata shard : getMissingShards()) { - stats.incrementBackgroundShardRecovery(); - ListenableFuture future = shardQueue.submit(new MissingShard(shard.getShardUuid(), shard.getCompressedSize(), shard.getXxhash64(), false)); - addExceptionCallback(future, t -> log.warn(t, "Error recovering shard: %s", shard.getShardUuid())); - } - } - catch (Throwable t) { - if (!executorService.isShutdown()) { - log.error(t, "Error creating shard recovery tasks"); - } - } - } - - private Set getMissingShards() - { - return shardManager.getNodeShards(nodeIdentifier).stream() - .filter(shard -> shardNeedsRecovery(shard.getShardUuid(), shard.getCompressedSize())) - .collect(toSet()); - } - - private boolean shardNeedsRecovery(UUID shardUuid, long shardSize) - { - File storageFile = storageService.getStorageFile(shardUuid); - return !storageFile.exists() || (storageFile.length() != shardSize); - } - - public Future recoverShard(UUID shardUuid) - throws ExecutionException - { - ShardMetadata shard = shardManager.getShard(shardUuid); - if (shard == null) { - throw new TrinoException(RAPTOR_ERROR, "Shard does not exist in database: " + shardUuid); - } - stats.incrementActiveShardRecovery(); - return shardQueue.submit(new MissingShard(shardUuid, shard.getCompressedSize(), shard.getXxhash64(), true)); - } - - @VisibleForTesting - void restoreFromBackup(UUID shardUuid, long shardSize, OptionalLong shardXxhash64) - { - File storageFile = storageService.getStorageFile(shardUuid); - - if (!backupStore.get().shardExists(shardUuid)) { - stats.incrementShardRecoveryBackupNotFound(); - throw new TrinoException(RAPTOR_RECOVERY_ERROR, "No backup file found for shard: " + shardUuid); - } - - if (storageFile.exists()) { - if (!isFileCorrupt(storageFile, shardSize, shardXxhash64)) { - return; - } - stats.incrementCorruptLocalFile(); - quarantineFile(shardUuid, storageFile, "Local file is corrupt."); - } - - // create a temporary file in the staging directory - File stagingFile = temporarySuffix(storageService.getStagingFile(shardUuid)); - storageService.createParents(stagingFile); - - // copy to temporary file - log.info("Copying shard %s from backup...", shardUuid); - long start = System.nanoTime(); - - try { - backupStore.get().restoreShard(shardUuid, stagingFile); - } - catch (TrinoException e) { - stats.incrementShardRecoveryFailure(); - stagingFile.delete(); - throw e; - } - - Duration duration = nanosSince(start); - DataSize size = succinctBytes(stagingFile.length()); - DataSize rate = dataRate(size, duration).succinct(); - stats.addShardRecoveryDataRate(rate, size, duration); - - log.info("Copied shard %s from backup in %s (%s at %s/s)", shardUuid, duration, size, rate); - - // move to final location - storageService.createParents(storageFile); - try { - Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE); - } - catch (FileAlreadyExistsException e) { - // someone else already created it (should not happen, but safe to ignore) - } - catch (IOException e) { - stats.incrementShardRecoveryFailure(); - throw new TrinoException(RAPTOR_RECOVERY_ERROR, "Failed to move shard: " + shardUuid, e); - } - finally { - stagingFile.delete(); - } - - if (!storageFile.exists()) { - stats.incrementShardRecoveryFailure(); - throw new TrinoException(RAPTOR_RECOVERY_ERROR, "File does not exist after recovery: " + shardUuid); - } - - if (isFileCorrupt(storageFile, shardSize, shardXxhash64)) { - stats.incrementShardRecoveryFailure(); - stats.incrementCorruptRecoveredFile(); - quarantineFile(shardUuid, storageFile, "Local file is corrupt after recovery."); - throw new TrinoException(RAPTOR_BACKUP_CORRUPTION, "Backup is corrupt after read: " + shardUuid); - } - - stats.incrementShardRecoverySuccess(); - } - - private void quarantineFile(UUID shardUuid, File file, String message) - { - File quarantine = new File(storageService.getQuarantineFile(shardUuid).getPath() + ".corrupt"); - if (quarantine.exists()) { - log.warn("%s Quarantine already exists: %s", message, quarantine); - return; - } - - log.error("%s Quarantining corrupt file: %s", message, quarantine); - try { - Files.move(file.toPath(), quarantine.toPath(), ATOMIC_MOVE); - } - catch (IOException e) { - log.warn(e, "Quarantine of corrupt file failed: %s", quarantine); - file.delete(); - } - } - - private static boolean isFileCorrupt(File file, long size, OptionalLong xxhash64) - { - return (file.length() != size) || (xxhash64.isPresent() && (xxhash64(file) != xxhash64.getAsLong())); - } - - @VisibleForTesting - static class MissingShardComparator - implements Comparator - { - @Override - public int compare(MissingShardRunnable shard1, MissingShardRunnable shard2) - { - if (shard1.isActive() == shard2.isActive()) { - return 0; - } - return shard1.isActive() ? -1 : 1; - } - } - - interface MissingShardRunnable - extends Runnable - { - boolean isActive(); - } - - private class MissingShardRecovery - implements MissingShardRunnable - { - private final UUID shardUuid; - private final long shardSize; - private final OptionalLong shardXxhash64; - private final boolean active; - - public MissingShardRecovery(UUID shardUuid, long shardSize, OptionalLong shardXxhash64, boolean active) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.shardSize = shardSize; - this.shardXxhash64 = requireNonNull(shardXxhash64, "shardXxhash64 is null"); - this.active = active; - } - - @Override - public void run() - { - restoreFromBackup(shardUuid, shardSize, shardXxhash64); - } - - @Override - public boolean isActive() - { - return active; - } - } - - private static final class MissingShard - { - private final UUID shardUuid; - private final long shardSize; - private final OptionalLong shardXxhash64; - private final boolean active; - - public MissingShard(UUID shardUuid, long shardSize, OptionalLong shardXxhash64, boolean active) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.shardSize = shardSize; - this.shardXxhash64 = requireNonNull(shardXxhash64, "shardXxhash64 is null"); - this.active = active; - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public long getShardSize() - { - return shardSize; - } - - public OptionalLong getShardXxhash64() - { - return shardXxhash64; - } - - public boolean isActive() - { - return active; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - MissingShard other = (MissingShard) o; - return this.active == other.active && - Objects.equals(this.shardUuid, other.shardUuid); - } - - @Override - public int hashCode() - { - return Objects.hash(shardUuid, active); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuid", shardUuid) - .add("active", active) - .toString(); - } - } - - private class MissingShardsQueue - { - private final LoadingCache> queuedMissingShards; - - public MissingShardsQueue(PrioritizedFifoExecutor shardRecoveryExecutor) - { - requireNonNull(shardRecoveryExecutor, "shardRecoveryExecutor is null"); - this.queuedMissingShards = buildUnsafeCache(CacheBuilder.newBuilder(), new CacheLoader<>() - { - @Override - public ListenableFuture load(MissingShard missingShard) - { - MissingShardRecovery task = new MissingShardRecovery( - missingShard.getShardUuid(), - missingShard.getShardSize(), - missingShard.getShardXxhash64(), - missingShard.isActive()); - ListenableFuture future = shardRecoveryExecutor.submit(task); - // TODO (https://github.com/trinodb/trino/issues/10688) invalidation here races with `.load()` completion - future.addListener(() -> queuedMissingShards.invalidate(missingShard), directExecutor()); - return future; - } - }); - } - - // TODO (https://github.com/trinodb/trino/issues/10688) there is a load/invalidation race, so it's an unsafe suppression - @SuppressModernizer - private LoadingCache buildUnsafeCache(CacheBuilder cacheBuilder, CacheLoader cacheLoader) - { - return cacheBuilder.build(cacheLoader); - } - - public ListenableFuture submit(MissingShard shard) - throws ExecutionException - { - return queuedMissingShards.get(shard); - } - } - - static DataSize dataRate(DataSize size, Duration duration) - { - double rate = size.toBytes() / duration.getValue(SECONDS); - if (Double.isNaN(rate) || Double.isInfinite(rate)) { - rate = 0; - } - return succinctBytes(Math.round(rate)); - } - - private static File temporarySuffix(File file) - { - return new File(file.getPath() + ".tmp-" + UUID.randomUUID()); - } - - @Managed - @Flatten - public ShardRecoveryStats getStats() - { - return stats; - } - - private static FutureCallback failureCallback(Consumer callback) - { - return new FutureCallback<>() - { - @Override - public void onSuccess(T result) {} - - @Override - public void onFailure(Throwable throwable) - { - callback.accept(throwable); - } - }; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryStats.java deleted file mode 100644 index ff25a862cdb4..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryStats.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.errorprone.annotations.ThreadSafe; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -@ThreadSafe -public class ShardRecoveryStats -{ - private final CounterStat activeShardRecovery = new CounterStat(); - private final CounterStat backgroundShardRecovery = new CounterStat(); - private final CounterStat shardRecoverySuccess = new CounterStat(); - private final CounterStat shardRecoveryFailure = new CounterStat(); - private final CounterStat shardRecoveryBackupNotFound = new CounterStat(); - - private final DistributionStat shardRecoveryShardSizeBytes = new DistributionStat(); - private final DistributionStat shardRecoveryTimeInMilliSeconds = new DistributionStat(); - private final DistributionStat shardRecoveryBytesPerSecond = new DistributionStat(); - - private final CounterStat corruptLocalFile = new CounterStat(); - private final CounterStat corruptRecoveredFile = new CounterStat(); - - public void incrementBackgroundShardRecovery() - { - backgroundShardRecovery.update(1); - } - - public void incrementActiveShardRecovery() - { - activeShardRecovery.update(1); - } - - public void incrementShardRecoveryBackupNotFound() - { - shardRecoveryBackupNotFound.update(1); - } - - public void incrementShardRecoveryFailure() - { - shardRecoveryFailure.update(1); - } - - public void incrementShardRecoverySuccess() - { - shardRecoverySuccess.update(1); - } - - public void addShardRecoveryDataRate(DataSize rate, DataSize size, Duration duration) - { - shardRecoveryBytesPerSecond.add(rate.toBytes()); - shardRecoveryShardSizeBytes.add(size.toBytes()); - shardRecoveryTimeInMilliSeconds.add(duration.toMillis()); - } - - public void incrementCorruptLocalFile() - { - corruptLocalFile.update(1); - } - - public void incrementCorruptRecoveredFile() - { - corruptRecoveredFile.update(1); - } - - @Managed - @Nested - public CounterStat getActiveShardRecovery() - { - return activeShardRecovery; - } - - @Managed - @Nested - public CounterStat getBackgroundShardRecovery() - { - return backgroundShardRecovery; - } - - @Managed - @Nested - public CounterStat getShardRecoverySuccess() - { - return shardRecoverySuccess; - } - - @Managed - @Nested - public CounterStat getShardRecoveryFailure() - { - return shardRecoveryFailure; - } - - @Managed - @Nested - public CounterStat getShardRecoveryBackupNotFound() - { - return shardRecoveryBackupNotFound; - } - - @Managed - @Nested - public DistributionStat getShardRecoveryBytesPerSecond() - { - return shardRecoveryBytesPerSecond; - } - - @Managed - @Nested - public DistributionStat getShardRecoveryTimeInMilliSeconds() - { - return shardRecoveryTimeInMilliSeconds; - } - - @Managed - @Nested - public DistributionStat getShardRecoveryShardSizeBytes() - { - return shardRecoveryShardSizeBytes; - } - - @Managed - @Nested - public CounterStat getCorruptLocalFile() - { - return corruptLocalFile; - } - - @Managed - @Nested - public CounterStat getCorruptRecoveredFile() - { - return corruptRecoveredFile; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRewriter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRewriter.java deleted file mode 100644 index 8d1645c78849..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRewriter.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.airlift.slice.Slice; - -import java.util.BitSet; -import java.util.Collection; -import java.util.concurrent.CompletableFuture; - -public interface ShardRewriter -{ - CompletableFuture> rewrite(BitSet rowsToDelete); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java deleted file mode 100644 index 4909d56e2b8c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.orc.OrcColumn; -import io.trino.orc.OrcPredicate; -import io.trino.orc.OrcReader; -import io.trino.orc.OrcRecordReader; -import io.trino.plugin.raptor.legacy.metadata.ColumnStats; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.VarcharType; - -import java.io.IOException; -import java.util.List; -import java.util.Optional; - -import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static io.trino.orc.OrcReader.INITIAL_BATCH_SIZE; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.toOrcFileType; -import static java.lang.Double.isInfinite; -import static java.lang.Double.isNaN; -import static org.joda.time.DateTimeZone.UTC; - -public final class ShardStats -{ - /** - * Maximum length of a binary value stored in an index. - */ - public static final int MAX_BINARY_INDEX_SIZE = 100; - - private ShardStats() {} - - public static Slice truncateIndexValue(Slice slice) - { - if (slice.length() > MAX_BINARY_INDEX_SIZE) { - return slice.slice(0, MAX_BINARY_INDEX_SIZE); - } - return slice; - } - - public static Optional computeColumnStats(OrcReader orcReader, long columnId, Type type, TypeManager typeManager) - throws IOException - { - return Optional.ofNullable(doComputeColumnStats(orcReader, columnId, type, typeManager)); - } - - private static ColumnStats doComputeColumnStats(OrcReader orcReader, long columnId, Type type, TypeManager typeManager) - throws IOException - { - OrcColumn column = getColumn(orcReader.getRootColumn().getNestedColumns(), columnId); - Type columnType = toOrcFileType(type, typeManager); - OrcRecordReader reader = orcReader.createRecordReader( - ImmutableList.of(column), - ImmutableList.of(columnType), - OrcPredicate.TRUE, - UTC, - newSimpleAggregatedMemoryContext(), - INITIAL_BATCH_SIZE, - exception -> new TrinoException(RAPTOR_ERROR, "Error reading column: " + columnId, exception)); - - if (type.equals(BooleanType.BOOLEAN)) { - return indexBoolean(reader, columnId); - } - if (type.equals(BigintType.BIGINT) || - type.equals(DateType.DATE) || - type.equals(TimestampType.TIMESTAMP_MILLIS)) { - return indexLong(type, reader, columnId); - } - if (type.equals(DoubleType.DOUBLE)) { - return indexDouble(reader, columnId); - } - if (type instanceof VarcharType) { - return indexString(type, reader, columnId); - } - return null; - } - - private static OrcColumn getColumn(List columnNames, long columnId) - { - String columnName = String.valueOf(columnId); - return columnNames.stream() - .filter(column -> column.getColumnName().equals(columnName)) - .findFirst() - .orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "Missing column ID: " + columnId)); - } - - private static ColumnStats indexBoolean(OrcRecordReader reader, long columnId) - throws IOException - { - boolean minSet = false; - boolean maxSet = false; - boolean min = false; - boolean max = false; - - while (true) { - Page page = reader.nextPage(); - if (page == null) { - break; - } - Block block = page.getBlock(0).getLoadedBlock(); - - for (int i = 0; i < page.getPositionCount(); i++) { - if (block.isNull(i)) { - continue; - } - boolean value = BooleanType.BOOLEAN.getBoolean(block, i); - if (!minSet || Boolean.compare(value, min) < 0) { - minSet = true; - min = value; - } - if (!maxSet || Boolean.compare(value, max) > 0) { - maxSet = true; - max = value; - } - } - } - - return new ColumnStats(columnId, - minSet ? min : null, - maxSet ? max : null); - } - - private static ColumnStats indexLong(Type type, OrcRecordReader reader, long columnId) - throws IOException - { - boolean minSet = false; - boolean maxSet = false; - long min = 0; - long max = 0; - - while (true) { - Page page = reader.nextPage(); - if (page == null) { - break; - } - Block block = page.getBlock(0).getLoadedBlock(); - - for (int i = 0; i < page.getPositionCount(); i++) { - if (block.isNull(i)) { - continue; - } - long value = type.getLong(block, i); - if (!minSet || (value < min)) { - minSet = true; - min = value; - } - if (!maxSet || (value > max)) { - maxSet = true; - max = value; - } - } - } - - return new ColumnStats(columnId, - minSet ? min : null, - maxSet ? max : null); - } - - private static ColumnStats indexDouble(OrcRecordReader reader, long columnId) - throws IOException - { - boolean minSet = false; - boolean maxSet = false; - double min = 0; - double max = 0; - - while (true) { - Page page = reader.nextPage(); - if (page == null) { - break; - } - Block block = page.getBlock(0).getLoadedBlock(); - - for (int i = 0; i < page.getPositionCount(); i++) { - if (block.isNull(i)) { - continue; - } - double value = DoubleType.DOUBLE.getDouble(block, i); - if (isNaN(value)) { - continue; - } - if (value == -0.0) { - value = 0.0; - } - if (!minSet || (value < min)) { - minSet = true; - min = value; - } - if (!maxSet || (value > max)) { - maxSet = true; - max = value; - } - } - } - - if (isInfinite(min)) { - minSet = false; - } - if (isInfinite(max)) { - maxSet = false; - } - - return new ColumnStats(columnId, - minSet ? min : null, - maxSet ? max : null); - } - - private static ColumnStats indexString(Type type, OrcRecordReader reader, long columnId) - throws IOException - { - boolean minSet = false; - boolean maxSet = false; - Slice min = null; - Slice max = null; - - while (true) { - Page page = reader.nextPage(); - if (page == null) { - break; - } - Block block = page.getBlock(0).getLoadedBlock(); - - for (int i = 0; i < page.getPositionCount(); i++) { - if (block.isNull(i)) { - continue; - } - Slice slice = type.getSlice(block, i); - slice = truncateIndexValue(slice); - if (!minSet || (slice.compareTo(min) < 0)) { - minSet = true; - min = slice; - } - if (!maxSet || (slice.compareTo(max) > 0)) { - maxSet = true; - max = slice; - } - } - } - - return new ColumnStats(columnId, - minSet ? min.toStringUtf8() : null, - maxSet ? max.toStringUtf8() : null); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManager.java deleted file mode 100644 index 628ef717a4e2..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManager.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.trino.orc.OrcReaderOptions; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.OptionalInt; -import java.util.UUID; - -public interface StorageManager -{ - ConnectorPageSource getPageSource( - UUID shardUuid, - OptionalInt bucketNumber, - List columnIds, - List columnTypes, - TupleDomain effectivePredicate, - OrcReaderOptions orcReaderOptions); - - StoragePageSink createStoragePageSink( - long transactionId, - OptionalInt bucketNumber, - List columnIds, - List columnTypes, - boolean checkSpace); - - ShardRewriter createShardRewriter( - long transactionId, - OptionalInt bucketNumber, - UUID shardUuid); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManagerConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManagerConfig.java deleted file mode 100644 index 2ce5a87410cb..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManagerConfig.java +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; -import io.airlift.configuration.LegacyConfig; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.airlift.units.MaxDataSize; -import io.airlift.units.MinDataSize; -import io.airlift.units.MinDuration; -import io.trino.orc.OrcReaderOptions; -import jakarta.validation.constraints.Max; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; - -import java.io.File; -import java.util.concurrent.TimeUnit; - -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static java.lang.Math.max; -import static java.lang.Runtime.getRuntime; - -@DefunctConfig({ - "storage.backup-directory", - "storage.shard-day-boundary-time-zone", -}) -public class StorageManagerConfig -{ - private File dataDirectory; - private DataSize minAvailableSpace = DataSize.ofBytes(0); - private Duration shardRecoveryTimeout = new Duration(30, TimeUnit.SECONDS); - private Duration missingShardDiscoveryInterval = new Duration(5, TimeUnit.MINUTES); - private boolean compactionEnabled = true; - private Duration compactionInterval = new Duration(1, TimeUnit.HOURS); - private Duration shardEjectorInterval = new Duration(4, TimeUnit.HOURS); - private OrcReaderOptions options = new OrcReaderOptions(); - private int deletionThreads = max(1, getRuntime().availableProcessors() / 2); - private int recoveryThreads = 10; - private int organizationThreads = 5; - private boolean organizationEnabled = true; - private Duration organizationDiscoveryInterval = new Duration(6, TimeUnit.HOURS); - private Duration organizationInterval = new Duration(7, TimeUnit.DAYS); - - private long maxShardRows = 1_000_000; - private DataSize maxShardSize = DataSize.of(256, MEGABYTE); - private DataSize maxBufferSize = DataSize.of(256, MEGABYTE); - private int oneSplitPerBucketThreshold; - - @NotNull - public File getDataDirectory() - { - return dataDirectory; - } - - @Config("storage.data-directory") - @ConfigDescription("Base directory to use for storing shard data") - public StorageManagerConfig setDataDirectory(File dataDirectory) - { - this.dataDirectory = dataDirectory; - return this; - } - - @NotNull - public DataSize getMinAvailableSpace() - { - return minAvailableSpace; - } - - @Config("storage.min-available-space") - @ConfigDescription("Minimum space that must be available on the data directory file system") - public StorageManagerConfig setMinAvailableSpace(DataSize minAvailableSpace) - { - this.minAvailableSpace = minAvailableSpace; - return this; - } - - public OrcReaderOptions toOrcReaderOptions() - { - return options; - } - - @NotNull - public DataSize getOrcMaxMergeDistance() - { - return options.getMaxMergeDistance(); - } - - @Config("storage.orc.max-merge-distance") - public StorageManagerConfig setOrcMaxMergeDistance(DataSize orcMaxMergeDistance) - { - options = options.withMaxMergeDistance(orcMaxMergeDistance); - return this; - } - - @NotNull - public DataSize getOrcMaxReadSize() - { - return options.getMaxBufferSize(); - } - - @Config("storage.orc.max-read-size") - public StorageManagerConfig setOrcMaxReadSize(DataSize orcMaxReadSize) - { - options = options.withMaxBufferSize(orcMaxReadSize); - return this; - } - - @NotNull - public DataSize getOrcStreamBufferSize() - { - return options.getStreamBufferSize(); - } - - @Config("storage.orc.stream-buffer-size") - public StorageManagerConfig setOrcStreamBufferSize(DataSize orcStreamBufferSize) - { - options = options.withStreamBufferSize(orcStreamBufferSize); - return this; - } - - @NotNull - public DataSize getOrcTinyStripeThreshold() - { - return options.getTinyStripeThreshold(); - } - - @Config("storage.orc.tiny-stripe-threshold") - public StorageManagerConfig setOrcTinyStripeThreshold(DataSize orcTinyStripeThreshold) - { - options = options.withTinyStripeThreshold(orcTinyStripeThreshold); - return this; - } - - @Deprecated - public boolean isOrcLazyReadSmallRanges() - { - return options.isLazyReadSmallRanges(); - } - - // TODO remove config option once efficacy is proven - @Deprecated - @Config("storage.orc.lazy-read-small-ranges") - public StorageManagerConfig setOrcLazyReadSmallRanges(boolean orcLazyReadSmallRanges) - { - options = options.withLazyReadSmallRanges(orcLazyReadSmallRanges); - return this; - } - - @Deprecated - public boolean isOrcNestedLazy() - { - return options.isNestedLazy(); - } - - // TODO remove config option once efficacy is proven - @Deprecated - @Config("storage.orc.nested-lazy") - public StorageManagerConfig setOrcNestedLazy(boolean nestedLazy) - { - options = options.withNestedLazy(nestedLazy); - return this; - } - - @Min(1) - public int getDeletionThreads() - { - return deletionThreads; - } - - @Config("storage.max-deletion-threads") - @ConfigDescription("Maximum number of threads to use for deletions") - public StorageManagerConfig setDeletionThreads(int deletionThreads) - { - this.deletionThreads = deletionThreads; - return this; - } - - @MinDuration("1s") - public Duration getShardRecoveryTimeout() - { - return shardRecoveryTimeout; - } - - @Config("storage.shard-recovery-timeout") - @ConfigDescription("Maximum time to wait for a shard to recover from backup while running a query") - public StorageManagerConfig setShardRecoveryTimeout(Duration shardRecoveryTimeout) - { - this.shardRecoveryTimeout = shardRecoveryTimeout; - return this; - } - - @MinDuration("1s") - public Duration getMissingShardDiscoveryInterval() - { - return missingShardDiscoveryInterval; - } - - @Config("storage.missing-shard-discovery-interval") - @ConfigDescription("How often to check the database and local file system missing shards") - public StorageManagerConfig setMissingShardDiscoveryInterval(Duration missingShardDiscoveryInterval) - { - this.missingShardDiscoveryInterval = missingShardDiscoveryInterval; - return this; - } - - @MinDuration("1s") - public Duration getCompactionInterval() - { - return compactionInterval; - } - - @Config("storage.compaction-interval") - @ConfigDescription("How often to check for local shards that need compaction") - public StorageManagerConfig setCompactionInterval(Duration compactionInterval) - { - this.compactionInterval = compactionInterval; - return this; - } - - @NotNull - @MinDuration("1s") - public Duration getOrganizationInterval() - { - return organizationInterval; - } - - @Config("storage.organization-interval") - @ConfigDescription("How long to wait between table organization iterations") - public StorageManagerConfig setOrganizationInterval(Duration organizationInterval) - { - this.organizationInterval = organizationInterval; - return this; - } - - @NotNull - @MinDuration("1s") - public Duration getOrganizationDiscoveryInterval() - { - return organizationDiscoveryInterval; - } - - @Config("storage.organization-discovery-interval") - @ConfigDescription("How long to wait between discovering tables that need to be organized") - public StorageManagerConfig setOrganizationDiscoveryInterval(Duration organizationDiscoveryInterval) - { - this.organizationDiscoveryInterval = organizationDiscoveryInterval; - return this; - } - - @MinDuration("5m") - public Duration getShardEjectorInterval() - { - return shardEjectorInterval; - } - - @Config("storage.ejector-interval") - @ConfigDescription("How often to check for local shards that need ejection to balance capacity") - public StorageManagerConfig setShardEjectorInterval(Duration shardEjectorInterval) - { - this.shardEjectorInterval = shardEjectorInterval; - return this; - } - - @Min(1) - public int getRecoveryThreads() - { - return recoveryThreads; - } - - @Config("storage.max-recovery-threads") - @ConfigDescription("Maximum number of threads to use for recovery") - public StorageManagerConfig setRecoveryThreads(int recoveryThreads) - { - this.recoveryThreads = recoveryThreads; - return this; - } - - @LegacyConfig("storage.max-compaction-threads") - @Config("storage.max-organization-threads") - @ConfigDescription("Maximum number of threads to use for organization") - public StorageManagerConfig setOrganizationThreads(int organizationThreads) - { - this.organizationThreads = organizationThreads; - return this; - } - - @Min(1) - public int getOrganizationThreads() - { - return organizationThreads; - } - - @Min(1) - @Max(1_000_000_000) - public long getMaxShardRows() - { - return maxShardRows; - } - - @Config("storage.max-shard-rows") - @ConfigDescription("Approximate maximum number of rows per shard") - public StorageManagerConfig setMaxShardRows(long maxShardRows) - { - this.maxShardRows = maxShardRows; - return this; - } - - @MinDataSize("1MB") - @MaxDataSize("1GB") - public DataSize getMaxShardSize() - { - return maxShardSize; - } - - @Config("storage.max-shard-size") - @ConfigDescription("Approximate maximum uncompressed size of a shard") - public StorageManagerConfig setMaxShardSize(DataSize maxShardSize) - { - this.maxShardSize = maxShardSize; - return this; - } - - @MinDataSize("1MB") - public DataSize getMaxBufferSize() - { - return maxBufferSize; - } - - @Config("storage.max-buffer-size") - @ConfigDescription("Maximum data to buffer before flushing to disk") - public StorageManagerConfig setMaxBufferSize(DataSize maxBufferSize) - { - this.maxBufferSize = maxBufferSize; - return this; - } - - public boolean isCompactionEnabled() - { - return compactionEnabled; - } - - @Config("storage.compaction-enabled") - public StorageManagerConfig setCompactionEnabled(boolean compactionEnabled) - { - this.compactionEnabled = compactionEnabled; - return this; - } - - public boolean isOrganizationEnabled() - { - return organizationEnabled; - } - - @Config("storage.organization-enabled") - public StorageManagerConfig setOrganizationEnabled(boolean organizationEnabled) - { - this.organizationEnabled = organizationEnabled; - return this; - } - - public int getOneSplitPerBucketThreshold() - { - return oneSplitPerBucketThreshold; - } - - @Config("storage.one-split-per-bucket-threshold") - @ConfigDescription("Experimental: Maximum bucket count at which to produce multiple splits per bucket") - public StorageManagerConfig setOneSplitPerBucketThreshold(int oneSplitPerBucketThreshold) - { - this.oneSplitPerBucketThreshold = oneSplitPerBucketThreshold; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java deleted file mode 100644 index 5c0d0c13ccbd..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.base.Ticker; -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Scopes; -import io.trino.plugin.raptor.legacy.backup.BackupManager; -import io.trino.plugin.raptor.legacy.metadata.AssignmentLimiter; -import io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager; -import io.trino.plugin.raptor.legacy.metadata.DatabaseShardRecorder; -import io.trino.plugin.raptor.legacy.metadata.MetadataConfig; -import io.trino.plugin.raptor.legacy.metadata.ShardCleaner; -import io.trino.plugin.raptor.legacy.metadata.ShardCleanerConfig; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; -import io.trino.plugin.raptor.legacy.storage.organization.JobFactory; -import io.trino.plugin.raptor.legacy.storage.organization.OrganizationJobFactory; -import io.trino.plugin.raptor.legacy.storage.organization.ShardCompactionManager; -import io.trino.plugin.raptor.legacy.storage.organization.ShardCompactor; -import io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizationManager; -import io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizer; - -import static io.airlift.configuration.ConfigBinder.configBinder; -import static org.weakref.jmx.guice.ExportBinder.newExporter; - -public class StorageModule - implements Module -{ - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(StorageManagerConfig.class); - configBinder(binder).bindConfig(BucketBalancerConfig.class); - configBinder(binder).bindConfig(ShardCleanerConfig.class); - configBinder(binder).bindConfig(MetadataConfig.class); - - binder.bind(Ticker.class).toInstance(Ticker.systemTicker()); - - binder.bind(StorageManager.class).to(RaptorStorageManager.class).in(Scopes.SINGLETON); - binder.bind(StorageService.class).to(FileStorageService.class).in(Scopes.SINGLETON); - binder.bind(ShardManager.class).to(DatabaseShardManager.class).in(Scopes.SINGLETON); - binder.bind(ShardRecorder.class).to(DatabaseShardRecorder.class).in(Scopes.SINGLETON); - binder.bind(DatabaseShardManager.class).in(Scopes.SINGLETON); - binder.bind(DatabaseShardRecorder.class).in(Scopes.SINGLETON); - binder.bind(ShardRecoveryManager.class).in(Scopes.SINGLETON); - binder.bind(BackupManager.class).in(Scopes.SINGLETON); - binder.bind(ShardCompactionManager.class).in(Scopes.SINGLETON); - binder.bind(ShardOrganizationManager.class).in(Scopes.SINGLETON); - binder.bind(ShardOrganizer.class).in(Scopes.SINGLETON); - binder.bind(JobFactory.class).to(OrganizationJobFactory.class).in(Scopes.SINGLETON); - binder.bind(ShardCompactor.class).in(Scopes.SINGLETON); - binder.bind(ShardEjector.class).in(Scopes.SINGLETON); - binder.bind(ShardCleaner.class).in(Scopes.SINGLETON); - binder.bind(BucketBalancer.class).in(Scopes.SINGLETON); - binder.bind(AssignmentLimiter.class).in(Scopes.SINGLETON); - - newExporter(binder).export(ShardRecoveryManager.class).withGeneratedName(); - newExporter(binder).export(BackupManager.class).withGeneratedName(); - newExporter(binder).export(StorageManager.class).as(generator -> generator.generatedNameOf(RaptorStorageManager.class)); - newExporter(binder).export(ShardCompactionManager.class).withGeneratedName(); - newExporter(binder).export(ShardOrganizer.class).withGeneratedName(); - newExporter(binder).export(ShardCompactor.class).withGeneratedName(); - newExporter(binder).export(ShardEjector.class).withGeneratedName(); - newExporter(binder).export(ShardCleaner.class).withGeneratedName(); - newExporter(binder).export(BucketBalancer.class).withGeneratedName(); - newExporter(binder).export(JobFactory.class).withGeneratedName(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StoragePageSink.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StoragePageSink.java deleted file mode 100644 index f999594f93ad..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StoragePageSink.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.spi.Page; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -public interface StoragePageSink -{ - default void appendPage(Page page) - { - appendPages(ImmutableList.of(page)); - } - - void appendPages(List pages); - - void appendPages(List pages, int[] pageIndexes, int[] positionIndexes); - - boolean isFull(); - - void flush(); - - CompletableFuture> commit(); - - void rollback(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageService.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageService.java deleted file mode 100644 index 05441257d529..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import java.io.File; -import java.util.Set; -import java.util.UUID; - -public interface StorageService -{ - void start(); - - long getAvailableBytes(); - - void createParents(File file); - - File getStorageFile(UUID shardUuid); - - File getStagingFile(UUID shardUuid); - - File getQuarantineFile(UUID shardUuid); - - Set getStorageShards(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/CompactionSetCreator.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/CompactionSetCreator.java deleted file mode 100644 index 728063b2adbf..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/CompactionSetCreator.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableSet; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.metadata.Table; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.createOrganizationSet; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getShardsByDaysBuckets; -import static java.util.Comparator.comparing; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toCollection; - -public class CompactionSetCreator -{ - private final DataSize maxShardSize; - private final long maxShardRows; - - public CompactionSetCreator(DataSize maxShardSize, long maxShardRows) - { - checkArgument(maxShardRows > 0, "maxShardRows must be > 0"); - - this.maxShardSize = requireNonNull(maxShardSize, "maxShardSize is null"); - this.maxShardRows = maxShardRows; - } - - // Expects a pre-filtered collection of shards. - // All shards provided to this method will be considered for creating a compaction set. - public Set createCompactionSets(Table tableInfo, Collection shards) - { - Collection> shardsByDaysBuckets = getShardsByDaysBuckets(tableInfo, shards); - - ImmutableSet.Builder compactionSets = ImmutableSet.builder(); - for (Collection shardInfos : shardsByDaysBuckets) { - compactionSets.addAll(buildCompactionSets(tableInfo, ImmutableSet.copyOf(shardInfos))); - } - return compactionSets.build(); - } - - private Set buildCompactionSets(Table tableInfo, Set shardIndexInfos) - { - long tableId = tableInfo.getTableId(); - List shards = shardIndexInfos.stream() - .sorted(getShardIndexInfoComparator(tableInfo)) - .collect(toCollection(ArrayList::new)); - - long consumedBytes = 0; - long consumedRows = 0; - ImmutableSet.Builder builder = ImmutableSet.builder(); - ImmutableSet.Builder compactionSets = ImmutableSet.builder(); - - for (ShardIndexInfo shard : shards) { - if (((consumedBytes + shard.getUncompressedSize()) > maxShardSize.toBytes()) || - (consumedRows + shard.getRowCount() > maxShardRows)) { - // Finalize this compaction set, and start a new one for the rest of the shards - Set shardsToCompact = builder.build(); - - if (shardsToCompact.size() > 1) { - compactionSets.add(createOrganizationSet(tableId, shardsToCompact)); - } - - builder = ImmutableSet.builder(); - consumedBytes = 0; - consumedRows = 0; - } - builder.add(shard); - consumedBytes += shard.getUncompressedSize(); - consumedRows += shard.getRowCount(); - } - - // create compaction set for the remaining shards of this day - Set shardsToCompact = builder.build(); - if (shardsToCompact.size() > 1) { - compactionSets.add(createOrganizationSet(tableId, shardsToCompact)); - } - return compactionSets.build(); - } - - private static Comparator getShardIndexInfoComparator(Table tableInfo) - { - if (tableInfo.getTemporalColumnId().isEmpty()) { - return comparing(ShardIndexInfo::getUncompressedSize); - } - - return comparing(info -> info.getTemporalRange().get(), - comparing(ShardRange::getMinTuple) - .thenComparing(ShardRange::getMaxTuple)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/JobFactory.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/JobFactory.java deleted file mode 100644 index 1e979e8ca1c3..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/JobFactory.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -public interface JobFactory -{ - Runnable create(OrganizationSet organizationSet); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJob.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJob.java deleted file mode 100644 index 6a73bd9ec446..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJob.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import io.airlift.log.Logger; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.plugin.raptor.legacy.metadata.TableMetadata; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.List; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST; -import static java.util.Collections.nCopies; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -class OrganizationJob - implements Runnable -{ - private static final Logger log = Logger.get(OrganizationJob.class); - - private final MetadataDao metadataDao; - private final ShardManager shardManager; - private final ShardCompactor compactor; - private final OrganizationSet organizationSet; - - public OrganizationJob(OrganizationSet organizationSet, MetadataDao metadataDao, ShardManager shardManager, ShardCompactor compactor) - { - this.metadataDao = requireNonNull(metadataDao, "metadataDao is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.compactor = requireNonNull(compactor, "compactor is null"); - this.organizationSet = requireNonNull(organizationSet, "organizationSet is null"); - } - - @Override - public void run() - { - try { - runJob(organizationSet.getTableId(), organizationSet.getBucketNumber(), organizationSet.getShards()); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private void runJob(long tableId, OptionalInt bucketNumber, Set shardUuids) - throws IOException - { - long transactionId = shardManager.beginTransaction(); - try { - runJob(transactionId, bucketNumber, tableId, shardUuids); - } - catch (Throwable e) { - shardManager.rollbackTransaction(transactionId); - throw e; - } - } - - private void runJob(long transactionId, OptionalInt bucketNumber, long tableId, Set shardUuids) - throws IOException - { - TableMetadata metadata = getTableMetadata(tableId); - List newShards = performCompaction(transactionId, bucketNumber, shardUuids, metadata); - log.info("Compacted shards %s into %s", shardUuids, newShards.stream().map(ShardInfo::getShardUuid).collect(toList())); - shardManager.replaceShardUuids(transactionId, tableId, metadata.getColumns(), shardUuids, newShards, OptionalLong.empty()); - } - - private TableMetadata getTableMetadata(long tableId) - { - List sortColumns = metadataDao.listSortColumns(tableId); - - List sortColumnIds = sortColumns.stream() - .map(TableColumn::getColumnId) - .collect(toList()); - - List columns = metadataDao.listTableColumns(tableId).stream() - .map(TableColumn::toColumnInfo) - .collect(toList()); - return new TableMetadata(tableId, columns, sortColumnIds); - } - - private List performCompaction(long transactionId, OptionalInt bucketNumber, Set shardUuids, TableMetadata tableMetadata) - throws IOException - { - if (tableMetadata.getSortColumnIds().isEmpty()) { - return compactor.compact(transactionId, bucketNumber, shardUuids, tableMetadata.getColumns()); - } - return compactor.compactSorted( - transactionId, - bucketNumber, - shardUuids, - tableMetadata.getColumns(), - tableMetadata.getSortColumnIds(), - nCopies(tableMetadata.getSortColumnIds().size(), ASC_NULLS_FIRST)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJobFactory.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJobFactory.java deleted file mode 100644 index fc9e69134186..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJobFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import org.jdbi.v3.core.Jdbi; - -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static java.util.Objects.requireNonNull; - -public class OrganizationJobFactory - implements JobFactory -{ - private final MetadataDao metadataDao; - private final ShardManager shardManager; - private final ShardCompactor compactor; - - @Inject - public OrganizationJobFactory(@ForMetadata Jdbi dbi, ShardManager shardManager, ShardCompactor compactor) - { - requireNonNull(dbi, "dbi is null"); - this.metadataDao = onDemandDao(dbi, MetadataDao.class); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.compactor = requireNonNull(compactor, "compactor is null"); - } - - @Override - public Runnable create(OrganizationSet organizationSet) - { - return new OrganizationJob(organizationSet, metadataDao, shardManager, compactor); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationSet.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationSet.java deleted file mode 100644 index 923e6f299ec5..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationSet.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import java.util.Objects; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class OrganizationSet -{ - private final long tableId; - private final Set shards; - private final OptionalInt bucketNumber; - - public OrganizationSet(long tableId, Set shards, OptionalInt bucketNumber) - { - this.tableId = tableId; - this.shards = requireNonNull(shards, "shards is null"); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - } - - public long getTableId() - { - return tableId; - } - - public Set getShards() - { - return shards; - } - - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OrganizationSet that = (OrganizationSet) o; - return tableId == that.tableId && - Objects.equals(shards, that.shards) && - Objects.equals(bucketNumber, that.bucketNumber); - } - - @Override - public int hashCode() - { - return Objects.hash(tableId, shards, bucketNumber); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("tableId", tableId) - .add("shards", shards) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactionManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactionManager.java deleted file mode 100644 index a8b554259b31..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactionManager.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Multimaps; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.NodeManager; -import io.trino.spi.type.Type; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.jdbi.v3.core.Jdbi; - -import java.util.Collection; -import java.util.List; -import java.util.Map.Entry; -import java.util.OptionalLong; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getOrganizationEligibleShards; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toSet; - -public class ShardCompactionManager -{ - private static final Logger log = Logger.get(ShardCompactionManager.class); - - private static final double FILL_FACTOR = 0.75; - - private final ScheduledExecutorService compactionDiscoveryService = newScheduledThreadPool(1, daemonThreadsNamed("shard-compaction-discovery")); - - private final AtomicBoolean discoveryStarted = new AtomicBoolean(); - - private final MetadataDao metadataDao; - private final ShardOrganizer organizer; - private final ShardManager shardManager; - private final String currentNodeIdentifier; - private final CompactionSetCreator compactionSetCreator; - - private final boolean compactionEnabled; - private final Duration compactionDiscoveryInterval; - private final DataSize maxShardSize; - private final long maxShardRows; - private final Jdbi dbi; - - @Inject - public ShardCompactionManager(@ForMetadata Jdbi dbi, - NodeManager nodeManager, - ShardManager shardManager, - ShardOrganizer organizer, - StorageManagerConfig config) - { - this(dbi, - nodeManager.getCurrentNode().getNodeIdentifier(), - shardManager, - organizer, - config.getCompactionInterval(), - config.getMaxShardSize(), - config.getMaxShardRows(), - config.isCompactionEnabled()); - } - - public ShardCompactionManager( - Jdbi dbi, - String currentNodeIdentifier, - ShardManager shardManager, - ShardOrganizer organizer, - Duration compactionDiscoveryInterval, - DataSize maxShardSize, - long maxShardRows, - boolean compactionEnabled) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.metadataDao = onDemandDao(dbi, MetadataDao.class); - - this.currentNodeIdentifier = requireNonNull(currentNodeIdentifier, "currentNodeIdentifier is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.organizer = requireNonNull(organizer, "organizer is null"); - this.compactionDiscoveryInterval = requireNonNull(compactionDiscoveryInterval, "compactionDiscoveryInterval is null"); - - checkArgument(maxShardSize.toBytes() > 0, "maxShardSize must be > 0"); - this.maxShardSize = requireNonNull(maxShardSize, "maxShardSize is null"); - - checkArgument(maxShardRows > 0, "maxShardRows must be > 0"); - this.maxShardRows = maxShardRows; - - this.compactionEnabled = compactionEnabled; - this.compactionSetCreator = new CompactionSetCreator(maxShardSize, maxShardRows); - } - - @PostConstruct - public void start() - { - if (!compactionEnabled) { - return; - } - - if (!discoveryStarted.getAndSet(true)) { - startDiscovery(); - } - } - - @PreDestroy - public void shutdown() - { - if (!compactionEnabled) { - return; - } - compactionDiscoveryService.shutdown(); - } - - private void startDiscovery() - { - compactionDiscoveryService.scheduleWithFixedDelay(() -> { - try { - // jitter to avoid overloading database - long interval = (long) compactionDiscoveryInterval.convertTo(SECONDS).getValue(); - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval)); - discoverShards(); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable t) { - log.error(t, "Error discovering shards to compact"); - } - }, 0, compactionDiscoveryInterval.toMillis(), TimeUnit.MILLISECONDS); - } - - private void discoverShards() - { - log.info("Discovering shards that need compaction..."); - Set allShards = shardManager.getNodeShards(currentNodeIdentifier); - ListMultimap tableShards = Multimaps.index(allShards, ShardMetadata::getTableId); - - for (Entry> entry : Multimaps.asMap(tableShards).entrySet()) { - long tableId = entry.getKey(); - if (!metadataDao.isCompactionEligible(tableId)) { - continue; - } - List shards = entry.getValue(); - Collection organizationSets = filterAndCreateCompactionSets(tableId, shards); - log.info("Created %s organization set(s) for table ID %s", organizationSets.size(), tableId); - - for (OrganizationSet set : organizationSets) { - organizer.enqueue(set); - } - } - } - - private Collection filterAndCreateCompactionSets(long tableId, Collection tableShards) - { - Table tableInfo = metadataDao.getTableInformation(tableId); - OptionalLong temporalColumnId = tableInfo.getTemporalColumnId(); - if (temporalColumnId.isPresent()) { - TableColumn tableColumn = metadataDao.getTableColumn(tableId, temporalColumnId.getAsLong()); - if (!isValidTemporalColumn(tableId, tableColumn.getDataType())) { - return ImmutableSet.of(); - } - } - - Set filteredShards = tableShards.stream() - .filter(this::needsCompaction) - .filter(shard -> !organizer.inProgress(shard.getShardUuid())) - .collect(toSet()); - - Collection shardIndexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, false); - if (tableInfo.getTemporalColumnId().isPresent()) { - Set temporalShards = shardIndexInfos.stream() - .filter(shard -> shard.getTemporalRange().isPresent()) - .collect(toSet()); - return compactionSetCreator.createCompactionSets(tableInfo, temporalShards); - } - - return compactionSetCreator.createCompactionSets(tableInfo, shardIndexInfos); - } - - private static boolean isValidTemporalColumn(long tableId, Type type) - { - if (!type.equals(DATE) && !type.equals(TIMESTAMP_MILLIS)) { - log.warn("Temporal column type of table ID %s set incorrectly to %s", tableId, type); - return false; - } - return true; - } - - private boolean needsCompaction(ShardMetadata shard) - { - if (shard.getUncompressedSize() < (FILL_FACTOR * maxShardSize.toBytes())) { - return true; - } - - if (shard.getRowCount() < (FILL_FACTOR * maxShardRows)) { - return true; - } - return false; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactor.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactor.java deleted file mode 100644 index 76a7473a2672..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactor.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; -import io.trino.orc.OrcReaderOptions; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.plugin.raptor.legacy.storage.StoragePageSink; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeOperators; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.io.Closeable; -import java.io.IOException; -import java.lang.invoke.MethodHandle; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.OptionalInt; -import java.util.PriorityQueue; -import java.util.Queue; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Throwables.throwIfUnchecked; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.BLOCK_POSITION; -import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.FAIL_ON_NULL; -import static io.trino.spi.function.InvocationConvention.simpleConvention; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public final class ShardCompactor -{ - private final StorageManager storageManager; - - private final CounterStat inputShards = new CounterStat(); - private final CounterStat outputShards = new CounterStat(); - private final DistributionStat inputShardsPerCompaction = new DistributionStat(); - private final DistributionStat outputShardsPerCompaction = new DistributionStat(); - private final DistributionStat compactionLatencyMillis = new DistributionStat(); - private final DistributionStat sortedCompactionLatencyMillis = new DistributionStat(); - private final OrcReaderOptions orcReaderOptions; - private final TypeOperators typeOperators; - - @Inject - public ShardCompactor(StorageManager storageManager, StorageManagerConfig config, TypeManager typeManager) - { - this(storageManager, - config.toOrcReaderOptions(), - typeManager.getTypeOperators()); - } - - public ShardCompactor(StorageManager storageManager, OrcReaderOptions orcReaderOptions, TypeOperators typeOperators) - { - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - this.orcReaderOptions = requireNonNull(orcReaderOptions, "orcReaderOptions is null"); - this.typeOperators = requireNonNull(typeOperators, "typeOperators is null"); - } - - public List compact(long transactionId, OptionalInt bucketNumber, Set uuids, List columns) - throws IOException - { - long start = System.nanoTime(); - List columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList()); - List columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList()); - - StoragePageSink storagePageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false); - - List shardInfos; - try { - shardInfos = compact(storagePageSink, bucketNumber, uuids, columnIds, columnTypes); - } - catch (IOException | RuntimeException e) { - storagePageSink.rollback(); - throw e; - } - - updateStats(uuids.size(), shardInfos.size(), nanosSince(start).toMillis()); - return shardInfos; - } - - private List compact(StoragePageSink storagePageSink, OptionalInt bucketNumber, Set uuids, List columnIds, List columnTypes) - throws IOException - { - for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), orcReaderOptions)) { - while (!pageSource.isFinished()) { - Page page = pageSource.getNextPage(); - if (isNullOrEmptyPage(page)) { - continue; - } - storagePageSink.appendPage(page); - if (storagePageSink.isFull()) { - storagePageSink.flush(); - } - } - } - } - return getFutureValue(storagePageSink.commit()); - } - - public List compactSorted(long transactionId, OptionalInt bucketNumber, Set uuids, List columns, List sortColumnIds, List sortOrders) - throws IOException - { - checkArgument(sortColumnIds.size() == sortOrders.size(), "sortColumnIds and sortOrders must be of the same size"); - - long start = System.nanoTime(); - - List columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList()); - List columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList()); - - checkArgument(ImmutableSet.copyOf(columnIds).containsAll(sortColumnIds), "sortColumnIds must be a subset of columnIds"); - - List sortIndexes = sortColumnIds.stream() - .map(columnIds::indexOf) - .collect(toList()); - - Queue rowSources = new PriorityQueue<>(); - StoragePageSink outputPageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false); - PageBuilder pageBuilder = new PageBuilder(columnTypes); - try { - for (UUID uuid : uuids) { - ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), orcReaderOptions); - SortedRowSource rowSource = new SortedRowSource(pageSource, columnTypes, sortIndexes, sortOrders, typeOperators); - rowSources.add(rowSource); - } - while (!rowSources.isEmpty()) { - SortedRowSource rowSource = rowSources.poll(); - if (!rowSource.hasNext()) { - // rowSource is empty, close it - rowSource.close(); - continue; - } - - rowSource.next().appendTo(pageBuilder); - - if (pageBuilder.isFull()) { - outputPageSink.appendPage(pageBuilder.build()); - pageBuilder.reset(); - } - - if (outputPageSink.isFull()) { - outputPageSink.flush(); - } - - rowSources.add(rowSource); - } - - if (!pageBuilder.isEmpty()) { - outputPageSink.appendPage(pageBuilder.build()); - } - - outputPageSink.flush(); - List shardInfos = getFutureValue(outputPageSink.commit()); - - updateStats(uuids.size(), shardInfos.size(), nanosSince(start).toMillis()); - - return shardInfos; - } - catch (IOException | RuntimeException e) { - outputPageSink.rollback(); - throw e; - } - finally { - rowSources.forEach(SortedRowSource::closeQuietly); - } - } - - private static class SortedRowSource - implements Iterator, Comparable, Closeable - { - private final ConnectorPageSource pageSource; - private final List sortIndexes; - private final List orderingOperators; - - private Page currentPage; - private int currentPosition; - - public SortedRowSource(ConnectorPageSource pageSource, List columnTypes, List sortIndexes, List sortOrders, TypeOperators typeOperators) - { - this.pageSource = requireNonNull(pageSource, "pageSource is null"); - this.sortIndexes = ImmutableList.copyOf(requireNonNull(sortIndexes, "sortIndexes is null")); - requireNonNull(columnTypes, "columnTypes is null"); - requireNonNull(sortOrders, "sortOrders is null"); - - ImmutableList.Builder orderingOperators = ImmutableList.builder(); - for (int index = 0; index < sortIndexes.size(); index++) { - Type type = columnTypes.get(sortIndexes.get(index)); - SortOrder sortOrder = sortOrders.get(index); - orderingOperators.add(typeOperators.getOrderingOperator(type, sortOrder, simpleConvention(FAIL_ON_NULL, BLOCK_POSITION, BLOCK_POSITION))); - } - this.orderingOperators = orderingOperators.build(); - - currentPage = pageSource.getNextPage(); - currentPosition = 0; - } - - @Override - public boolean hasNext() - { - if (hasMorePositions(currentPage, currentPosition)) { - return true; - } - - Page page = getNextPage(pageSource); - if (isNullOrEmptyPage(page)) { - return false; - } - currentPage = page.getLoadedPage(); - currentPosition = 0; - return true; - } - - private static Page getNextPage(ConnectorPageSource pageSource) - { - Page page = null; - while (isNullOrEmptyPage(page) && !pageSource.isFinished()) { - page = pageSource.getNextPage(); - if (page != null) { - page = page.getLoadedPage(); - } - } - return page; - } - - @Override - public Row next() - { - if (!hasNext()) { - throw new NoSuchElementException(); - } - - Row row = new Row(currentPage, currentPosition); - currentPosition++; - return row; - } - - @Override - public int compareTo(SortedRowSource other) - { - if (!hasNext()) { - return 1; - } - - if (!other.hasNext()) { - return -1; - } - - try { - for (int i = 0; i < sortIndexes.size(); i++) { - int channel = sortIndexes.get(i); - - Block leftBlock = currentPage.getBlock(channel); - int leftBlockPosition = currentPosition; - - Block rightBlock = other.currentPage.getBlock(channel); - int rightBlockPosition = other.currentPosition; - - MethodHandle comparator = orderingOperators.get(i); - int compare = (int) comparator.invokeExact(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition); - if (compare != 0) { - return compare; - } - } - return 0; - } - catch (Throwable throwable) { - throwIfUnchecked(throwable); - throw new TrinoException(GENERIC_INTERNAL_ERROR, throwable); - } - } - - private static boolean hasMorePositions(Page currentPage, int currentPosition) - { - return currentPage != null && currentPosition < currentPage.getPositionCount(); - } - - void closeQuietly() - { - try { - close(); - } - catch (IOException _) { - } - } - - @Override - public void close() - throws IOException - { - pageSource.close(); - } - } - - private static class Row - { - private final Page page; - private final int position; - - public Row(Page page, int position) - { - this.page = requireNonNull(page, "page is null"); - this.position = position; - } - - public void appendTo(PageBuilder pageBuilder) - { - pageBuilder.declarePosition(); - for (int channel = 0; channel < page.getChannelCount(); channel++) { - Block block = page.getBlock(channel); - BlockBuilder output = pageBuilder.getBlockBuilder(channel); - pageBuilder.getType(channel).appendTo(block, position, output); - } - } - } - - private static boolean isNullOrEmptyPage(Page nextPage) - { - return nextPage == null || nextPage.getPositionCount() == 0; - } - - private void updateStats(int inputShardsCount, int outputShardsCount, long latency) - { - inputShards.update(inputShardsCount); - outputShards.update(outputShardsCount); - - inputShardsPerCompaction.add(inputShardsCount); - outputShardsPerCompaction.add(outputShardsCount); - - compactionLatencyMillis.add(latency); - } - - @Managed - @Nested - public CounterStat getInputShards() - { - return inputShards; - } - - @Managed - @Nested - public CounterStat getOutputShards() - { - return outputShards; - } - - @Managed - @Nested - public DistributionStat getInputShardsPerCompaction() - { - return inputShardsPerCompaction; - } - - @Managed - @Nested - public DistributionStat getOutputShardsPerCompaction() - { - return outputShardsPerCompaction; - } - - @Managed - @Nested - public DistributionStat getCompactionLatencyMillis() - { - return compactionLatencyMillis; - } - - @Managed - @Nested - public DistributionStat getSortedCompactionLatencyMillis() - { - return sortedCompactionLatencyMillis; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardIndexInfo.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardIndexInfo.java deleted file mode 100644 index cf7d03a01dda..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardIndexInfo.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardIndexInfo -{ - private final long tableId; - private final OptionalInt bucketNumber; - private final UUID shardUuid; - private final long rowCount; - private final long uncompressedSize; - private final Optional sortRange; - private final Optional temporalRange; - - public ShardIndexInfo( - long tableId, - OptionalInt bucketNumber, - UUID shardUuid, - long rowCount, - long uncompressedSize, - Optional sortRange, - Optional temporalRange) - { - this.tableId = tableId; - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.rowCount = rowCount; - this.uncompressedSize = uncompressedSize; - this.sortRange = requireNonNull(sortRange, "sortRange is null"); - this.temporalRange = requireNonNull(temporalRange, "temporalRange is null"); - } - - public long getTableId() - { - return tableId; - } - - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public long getRowCount() - { - return rowCount; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - - public Optional getSortRange() - { - return sortRange; - } - - public Optional getTemporalRange() - { - return temporalRange; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ShardIndexInfo that = (ShardIndexInfo) o; - return tableId == that.tableId && - rowCount == that.rowCount && - uncompressedSize == that.uncompressedSize && - Objects.equals(bucketNumber, that.bucketNumber) && - Objects.equals(shardUuid, that.shardUuid) && - Objects.equals(sortRange, that.sortRange) && - Objects.equals(temporalRange, that.temporalRange); - } - - @Override - public int hashCode() - { - return Objects.hash(tableId, bucketNumber, shardUuid, rowCount, uncompressedSize, sortRange, temporalRange); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("tableId", tableId) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .add("shardUuid", shardUuid) - .add("rowCount", rowCount) - .add("uncompressedSize", uncompressedSize) - .add("sortRange", sortRange.orElse(null)) - .add("temporalRange", temporalRange.orElse(null)) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizationManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizationManager.java deleted file mode 100644 index cb6e1792506f..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizationManager.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.NodeManager; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.jdbi.v3.core.Jdbi; - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.collect.Sets.difference; -import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.createOrganizationSet; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getOrganizationEligibleShards; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getShardsByDaysBuckets; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static java.lang.Math.max; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; - -public class ShardOrganizationManager -{ - private static final Logger log = Logger.get(ShardOrganizationManager.class); - - private final ScheduledExecutorService discoveryService = newScheduledThreadPool(1, daemonThreadsNamed("shard-organization-discovery")); - private final AtomicBoolean started = new AtomicBoolean(); - - private final Jdbi dbi; - private final MetadataDao metadataDao; - private final ShardOrganizerDao organizerDao; - private final ShardManager shardManager; - - private final boolean enabled; - private final long organizationIntervalMillis; - private final long organizationDiscoveryIntervalMillis; - - private final String currentNodeIdentifier; - private final ShardOrganizer organizer; - - private final Set tablesInProgress = newConcurrentHashSet(); - - @Inject - public ShardOrganizationManager( - @ForMetadata Jdbi dbi, - NodeManager nodeManager, - ShardManager shardManager, - ShardOrganizer organizer, - StorageManagerConfig config) - { - this(dbi, - nodeManager.getCurrentNode().getNodeIdentifier(), - shardManager, - organizer, - config.isOrganizationEnabled(), - config.getOrganizationInterval(), - config.getOrganizationDiscoveryInterval()); - } - - public ShardOrganizationManager( - Jdbi dbi, - String currentNodeIdentifier, - ShardManager shardManager, - ShardOrganizer organizer, - boolean enabled, - Duration organizationInterval, - Duration organizationDiscoveryInterval) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.metadataDao = onDemandDao(dbi, MetadataDao.class); - this.organizerDao = onDemandDao(dbi, ShardOrganizerDao.class); - - this.organizer = requireNonNull(organizer, "organizer is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.currentNodeIdentifier = requireNonNull(currentNodeIdentifier, "currentNodeIdentifier is null"); - - this.enabled = enabled; - - requireNonNull(organizationInterval, "organizationInterval is null"); - this.organizationIntervalMillis = max(1, organizationInterval.roundTo(MILLISECONDS)); - this.organizationDiscoveryIntervalMillis = max(1, organizationDiscoveryInterval.roundTo(MILLISECONDS)); - } - - @PostConstruct - public void start() - { - if (!enabled || started.getAndSet(true)) { - return; - } - - startDiscovery(); - } - - @PreDestroy - public void shutdown() - { - discoveryService.shutdownNow(); - } - - private void startDiscovery() - { - discoveryService.scheduleWithFixedDelay(() -> { - try { - // jitter to avoid overloading database and overloading the backup store - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, organizationDiscoveryIntervalMillis)); - - log.info("Running shard organizer..."); - submitJobs(discoverAndInitializeTablesToOrganize()); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable t) { - log.error(t, "Error running shard organizer"); - } - }, 0, organizationDiscoveryIntervalMillis, TimeUnit.MILLISECONDS); - } - - @VisibleForTesting - Set discoverAndInitializeTablesToOrganize() - { - Set enabledTableIds = metadataDao.getOrganizationEligibleTables(); - - Set tableOrganizationInfo = organizerDao.getNodeTableOrganizationInfo(currentNodeIdentifier); - Map organizationInfos = Maps.uniqueIndex(tableOrganizationInfo, TableOrganizationInfo::getTableId); - - // If this is the first time organizing a table, initialize the organization info for it - difference(enabledTableIds, organizationInfos.keySet()) - .forEach(tableId -> organizerDao.insertNode(currentNodeIdentifier, tableId)); - - ImmutableSet.Builder tableIds = ImmutableSet.builder(); - for (Long tableId : enabledTableIds) { - TableOrganizationInfo info = organizationInfos.get(tableId); - if (info == null || shouldRunOrganization(info)) { - tableIds.add(tableId); - } - } - return tableIds.build(); - } - - private void submitJobs(Set tableIds) - { - tableIds.forEach(this::runOrganization); - } - - private void runOrganization(long tableId) - { - Set shardMetadatas = shardManager.getNodeShards(currentNodeIdentifier, tableId); - Table tableInfo = metadataDao.getTableInformation(tableId); - Set filteredShards = shardMetadatas.stream() - .filter(shard -> !organizer.inProgress(shard.getShardUuid())) - .collect(toSet()); - - Collection indexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, true); - Set organizationSets = createOrganizationSets(tableInfo, indexInfos); - - if (organizationSets.isEmpty()) { - return; - } - - log.info("Created %s organization set(s) from %s shards for table ID %s", organizationSets.size(), filteredShards.size(), tableId); - - long lastStartTime = System.currentTimeMillis(); - tablesInProgress.add(tableId); - - ImmutableList.Builder> futures = ImmutableList.builder(); - for (OrganizationSet organizationSet : organizationSets) { - futures.add(organizer.enqueue(organizationSet)); - } - allAsList(futures.build()) - .whenComplete((value, throwable) -> { - tablesInProgress.remove(tableId); - organizerDao.updateLastStartTime(currentNodeIdentifier, tableId, lastStartTime); - }); - } - - private boolean shouldRunOrganization(TableOrganizationInfo info) - { - // skip if organization is in progress for this table - if (tablesInProgress.contains(info.getTableId())) { - return false; - } - - if (info.getLastStartTimeMillis().isEmpty()) { - return true; - } - - return (System.currentTimeMillis() - info.getLastStartTimeMillis().getAsLong()) >= organizationIntervalMillis; - } - - @VisibleForTesting - static Set createOrganizationSets(Table tableInfo, Collection shards) - { - return getShardsByDaysBuckets(tableInfo, shards).stream() - .map(indexInfos -> getOverlappingOrganizationSets(tableInfo, indexInfos)) - .flatMap(Collection::stream) - .collect(toSet()); - } - - private static Set getOverlappingOrganizationSets(Table tableInfo, Collection shards) - { - if (shards.size() <= 1) { - return ImmutableSet.of(); - } - - // Sort by low marker for the range - List sortedShards = shards.stream() - .sorted((o1, o2) -> { - ShardRange sortRange1 = o1.getSortRange().get(); - ShardRange sortRange2 = o2.getSortRange().get(); - return ComparisonChain.start() - .compare(sortRange1.getMinTuple(), sortRange2.getMinTuple()) - .compare(sortRange2.getMaxTuple(), sortRange1.getMaxTuple()) - .result(); - }) - .collect(toList()); - - Set organizationSets = new HashSet<>(); - ImmutableSet.Builder builder = ImmutableSet.builder(); - - builder.add(sortedShards.get(0)); - int previousRange = 0; - int nextRange = previousRange + 1; - while (nextRange < sortedShards.size()) { - ShardRange sortRange1 = sortedShards.get(previousRange).getSortRange().get(); - ShardRange sortRange2 = sortedShards.get(nextRange).getSortRange().get(); - - if (sortRange1.overlaps(sortRange2) && !sortRange1.adjacent(sortRange2)) { - builder.add(sortedShards.get(nextRange)); - if (!sortRange1.encloses(sortRange2)) { - previousRange = nextRange; - } - } - else { - Set indexInfos = builder.build(); - if (indexInfos.size() > 1) { - organizationSets.add(createOrganizationSet(tableInfo.getTableId(), indexInfos)); - } - builder = ImmutableSet.builder(); - previousRange = nextRange; - builder.add(sortedShards.get(previousRange)); - } - nextRange++; - } - - Set indexInfos = builder.build(); - if (indexInfos.size() > 1) { - organizationSets.add(createOrganizationSet(tableInfo.getTableId(), indexInfos)); - } - return organizationSets; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizer.java deleted file mode 100644 index ba9512c9a027..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizer.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.inject.Inject; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadPoolExecutor; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Sets.newConcurrentHashSet; -import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.runAsync; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class ShardOrganizer -{ - private static final Logger log = Logger.get(ShardOrganizer.class); - - private final ExecutorService executorService; - private final ThreadPoolExecutorMBean executorMBean; - - // Tracks shards that are scheduled for compaction so that we do not schedule them more than once - private final Set shardsInProgress = newConcurrentHashSet(); - private final JobFactory jobFactory; - private final CounterStat successCount = new CounterStat(); - private final CounterStat failureCount = new CounterStat(); - - @Inject - public ShardOrganizer(JobFactory jobFactory, StorageManagerConfig config) - { - this(jobFactory, config.getOrganizationThreads()); - } - - public ShardOrganizer(JobFactory jobFactory, int threads) - { - checkArgument(threads > 0, "threads must be > 0"); - this.jobFactory = requireNonNull(jobFactory, "jobFactory is null"); - this.executorService = newFixedThreadPool(threads, daemonThreadsNamed("shard-organizer-%s")); - this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) executorService); - } - - @PreDestroy - public void shutdown() - { - shutdownAndAwaitTermination(executorService, 10, SECONDS); - } - - public CompletableFuture enqueue(OrganizationSet organizationSet) - { - shardsInProgress.addAll(organizationSet.getShards()); - return runAsync(jobFactory.create(organizationSet), executorService) - .whenComplete((none, throwable) -> { - shardsInProgress.removeAll(organizationSet.getShards()); - if (throwable == null) { - successCount.update(1); - } - else { - log.warn(throwable, "Error running organization job"); - failureCount.update(1); - } - }); - } - - public boolean inProgress(UUID shardUuid) - { - return shardsInProgress.contains(shardUuid); - } - - @Managed - @Nested - public ThreadPoolExecutorMBean getExecutor() - { - return executorMBean; - } - - @Managed - public int getShardsInProgress() - { - return shardsInProgress.size(); - } - - @Managed - @Nested - public CounterStat getSuccessCount() - { - return successCount; - } - - @Managed - @Nested - public CounterStat getFailureCount() - { - return failureCount; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerDao.java deleted file mode 100644 index 7acd59874e4e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerDao.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.util.Set; - -@RegisterConstructorMapper(TableOrganizationInfo.class) -public interface ShardOrganizerDao -{ - @SqlUpdate("INSERT INTO shard_organizer_jobs (node_identifier, table_id, last_start_time)\n" + - "VALUES (:nodeIdentifier, :tableId, NULL)") - void insertNode(String nodeIdentifier, long tableId); - - @SqlUpdate("UPDATE shard_organizer_jobs SET last_start_time = :lastStartTime\n" + - " WHERE node_identifier = :nodeIdentifier\n" + - " AND table_id = :tableId") - void updateLastStartTime( - String nodeIdentifier, - long tableId, - long lastStartTime); - - @SqlQuery("SELECT table_id, last_start_time\n" + - " FROM shard_organizer_jobs\n" + - " WHERE node_identifier = :nodeIdentifier") - Set getNodeTableOrganizationInfo(String nodeIdentifier); - - @SqlUpdate("DELETE FROM shard_organizer_jobs WHERE table_id = :tableId") - void dropOrganizerJobs(long tableId); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerUtil.java deleted file mode 100644 index ae26b616a553..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerUtil.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Multimaps; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Jdbi; - -import java.sql.Connection; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.collect.Iterables.partition; -import static com.google.common.collect.Maps.uniqueIndex; -import static io.airlift.slice.Slices.wrappedBuffer; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.storage.ColumnIndexStatsUtils.jdbcType; -import static java.lang.String.format; -import static java.util.Collections.nCopies; -import static java.util.stream.Collectors.toSet; - -public final class ShardOrganizerUtil -{ - private ShardOrganizerUtil() {} - - public static Collection getOrganizationEligibleShards( - Jdbi dbi, - MetadataDao metadataDao, - Table tableInfo, - Collection shards, - boolean includeSortColumns) - { - Map shardsById = uniqueIndex(shards, ShardMetadata::getShardId); - long tableId = tableInfo.getTableId(); - - ImmutableList.Builder columnsBuilder = ImmutableList.builder(); - columnsBuilder.add("shard_id"); - - // include temporal columns if present - Optional temporalColumn = Optional.empty(); - if (tableInfo.getTemporalColumnId().isPresent()) { - long temporalColumnId = tableInfo.getTemporalColumnId().getAsLong(); - temporalColumn = Optional.of(metadataDao.getTableColumn(tableId, temporalColumnId)); - columnsBuilder.add(minColumn(temporalColumnId), maxColumn(temporalColumnId)); - } - - // include sort columns if needed - Optional> sortColumns = Optional.empty(); - if (includeSortColumns) { - sortColumns = Optional.of(metadataDao.listSortColumns(tableId)); - for (TableColumn column : sortColumns.get()) { - columnsBuilder.add(minColumn(column.getColumnId()), maxColumn(column.getColumnId())); - } - } - String columnToSelect = Joiner.on(",\n").join(columnsBuilder.build()); - - ImmutableList.Builder indexInfoBuilder = ImmutableList.builder(); - try (Connection connection = dbi.open().getConnection()) { - for (List partitionedShards : partition(shards, 1000)) { - String shardIds = Joiner.on(",").join(nCopies(partitionedShards.size(), "?")); - - String sql = format("" + - "SELECT %s\n" + - "FROM %s\n" + - "WHERE shard_id IN (%s)", - columnToSelect, shardIndexTable(tableId), shardIds); - - try (PreparedStatement statement = connection.prepareStatement(sql)) { - for (int i = 0; i < partitionedShards.size(); i++) { - statement.setLong(i + 1, partitionedShards.get(i).getShardId()); - } - try (ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - long shardId = resultSet.getLong("shard_id"); - - Optional sortRange = Optional.empty(); - if (includeSortColumns) { - sortRange = getShardRange(sortColumns.get(), resultSet); - if (sortRange.isEmpty()) { - continue; - } - } - Optional temporalRange = Optional.empty(); - if (temporalColumn.isPresent()) { - temporalRange = getShardRange(ImmutableList.of(temporalColumn.get()), resultSet); - if (temporalRange.isEmpty()) { - continue; - } - } - ShardMetadata shardMetadata = shardsById.get(shardId); - indexInfoBuilder.add(toShardIndexInfo(shardMetadata, temporalRange, sortRange)); - } - } - } - } - } - catch (SQLException e) { - throw new RuntimeException(e); - } - return indexInfoBuilder.build(); - } - - private static ShardIndexInfo toShardIndexInfo(ShardMetadata shardMetadata, Optional temporalRange, Optional sortRange) - { - return new ShardIndexInfo( - shardMetadata.getTableId(), - shardMetadata.getBucketNumber(), - shardMetadata.getShardUuid(), - shardMetadata.getRowCount(), - shardMetadata.getUncompressedSize(), - sortRange, - temporalRange); - } - - public static Collection> getShardsByDaysBuckets(Table tableInfo, Collection shards) - { - if (shards.isEmpty()) { - return ImmutableList.of(); - } - - // Neither bucketed nor temporal, no partitioning required - if (tableInfo.getBucketCount().isEmpty() && tableInfo.getTemporalColumnId().isEmpty()) { - return ImmutableList.of(shards); - } - - // if only bucketed, partition by bucket number - if (tableInfo.getBucketCount().isPresent() && tableInfo.getTemporalColumnId().isEmpty()) { - return Multimaps.index(shards, shard -> shard.getBucketNumber().getAsInt()).asMap().values(); - } - - // if temporal, partition into days first - ImmutableMultimap.Builder shardsByDaysBuilder = ImmutableMultimap.builder(); - shards.stream() - .filter(shard -> shard.getTemporalRange().isPresent()) - .forEach(shard -> { - long day = TemporalFunction.getDayFromRange(shard.getTemporalRange().get()); - shardsByDaysBuilder.put(day, shard); - }); - - Collection> byDays = shardsByDaysBuilder.build().asMap().values(); - - // if table is bucketed further partition by bucket number - if (tableInfo.getBucketCount().isEmpty()) { - return byDays; - } - - ImmutableList.Builder> sets = ImmutableList.builder(); - for (Collection s : byDays) { - sets.addAll(Multimaps.index(s, ShardIndexInfo::getBucketNumber).asMap().values()); - } - return sets.build(); - } - - private static Optional getShardRange(List columns, ResultSet resultSet) - throws SQLException - { - ImmutableList.Builder minValuesBuilder = ImmutableList.builder(); - ImmutableList.Builder maxValuesBuilder = ImmutableList.builder(); - ImmutableList.Builder typeBuilder = ImmutableList.builder(); - - for (TableColumn tableColumn : columns) { - long columnId = tableColumn.getColumnId(); - Type type = tableColumn.getDataType(); - - Object min = getValue(resultSet, type, minColumn(columnId)); - Object max = getValue(resultSet, type, maxColumn(columnId)); - - if (min == null || max == null) { - return Optional.empty(); - } - - minValuesBuilder.add(min); - maxValuesBuilder.add(max); - typeBuilder.add(type); - } - - List types = typeBuilder.build(); - Tuple minTuple = new Tuple(types, minValuesBuilder.build()); - Tuple maxTuple = new Tuple(types, maxValuesBuilder.build()); - - return Optional.of(ShardRange.of(minTuple, maxTuple)); - } - - private static Object getValue(ResultSet resultSet, Type type, String columnName) - throws SQLException - { - JDBCType jdbcType = jdbcType(type); - Object value = getValue(resultSet, type, columnName, jdbcType); - return resultSet.wasNull() ? null : value; - } - - private static Object getValue(ResultSet resultSet, Type type, String columnName, JDBCType jdbcType) - throws SQLException - { - return switch (jdbcType) { - case BOOLEAN -> resultSet.getBoolean(columnName); - case INTEGER -> resultSet.getInt(columnName); - case BIGINT -> resultSet.getLong(columnName); - case DOUBLE -> resultSet.getDouble(columnName); - case VARBINARY -> wrappedBuffer(resultSet.getBytes(columnName)).toStringUtf8(); - default -> throw new IllegalArgumentException("Unhandled type: " + type); - }; - } - - static OrganizationSet createOrganizationSet(long tableId, Set shardsToCompact) - { - Set uuids = shardsToCompact.stream() - .map(ShardIndexInfo::getShardUuid) - .collect(toSet()); - - Set bucketNumber = shardsToCompact.stream() - .map(ShardIndexInfo::getBucketNumber) - .collect(toSet()); - - checkArgument(bucketNumber.size() == 1); - return new OrganizationSet(tableId, uuids, getOnlyElement(bucketNumber)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardRange.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardRange.java deleted file mode 100644 index eec709716039..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardRange.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardRange -{ - private final Tuple minTuple; - private final Tuple maxTuple; - - public static ShardRange of(Tuple min, Tuple max) - { - return new ShardRange(min, max); - } - - private ShardRange(Tuple minTuple, Tuple maxTuple) - { - this.minTuple = requireNonNull(minTuple, "minTuple is null"); - this.maxTuple = requireNonNull(maxTuple, "maxTuple is null"); - } - - public Tuple getMinTuple() - { - return minTuple; - } - - public Tuple getMaxTuple() - { - return maxTuple; - } - - public boolean encloses(ShardRange other) - { - return this.getMinTuple().compareTo(other.getMinTuple()) <= 0 && - this.getMaxTuple().compareTo(other.getMaxTuple()) >= 0; - } - - public boolean overlaps(ShardRange other) - { - return this.getMinTuple().compareTo(other.getMaxTuple()) <= 0 && - other.getMinTuple().compareTo(this.getMaxTuple()) <= 0; - } - - public boolean adjacent(ShardRange other) - { - Object o1Min = this.getMinTuple().getValues().get(0); - Object o1Max = this.getMaxTuple().getValues().get(0); - - Object o2Min = other.getMinTuple().getValues().get(0); - Object o2Max = other.getMaxTuple().getValues().get(0); - - return o1Max.equals(o2Min) || o2Max.equals(o1Min); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ShardRange that = (ShardRange) o; - return Objects.equals(minTuple, that.minTuple) && - Objects.equals(maxTuple, that.maxTuple); - } - - @Override - public int hashCode() - { - return Objects.hash(minTuple, maxTuple); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("minTuple", minTuple) - .add("maxTuple", maxTuple) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TableOrganizationInfo.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TableOrganizationInfo.java deleted file mode 100644 index 907e4c0b7ae1..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TableOrganizationInfo.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import org.jdbi.v3.core.mapper.reflect.ColumnName; - -import java.util.OptionalLong; - -import static java.util.Objects.requireNonNull; - -public class TableOrganizationInfo -{ - private final long tableId; - private final OptionalLong lastStartTimeMillis; - - public TableOrganizationInfo(long tableId, @ColumnName("last_start_time") OptionalLong lastStartTimeMillis) - { - this.tableId = tableId; - this.lastStartTimeMillis = requireNonNull(lastStartTimeMillis, "lastStartTimeMillis is null"); - } - - public long getTableId() - { - return tableId; - } - - public OptionalLong getLastStartTimeMillis() - { - return lastStartTimeMillis; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TemporalFunction.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TemporalFunction.java deleted file mode 100644 index 94c35acdce91..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TemporalFunction.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import io.trino.spi.block.Block; -import io.trino.spi.type.Type; - -import java.time.Duration; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_DAY; -import static java.lang.Math.floorDiv; -import static java.lang.Math.toIntExact; - -public final class TemporalFunction -{ - private TemporalFunction() {} - - public static int getDay(Type type, Block block, int position) - { - if (type.equals(DATE)) { - return DATE.getInt(block, position); - } - - if (type.equals(TIMESTAMP_MILLIS)) { - long days = floorDiv(TIMESTAMP_MILLIS.getLong(block, position), MICROSECONDS_PER_DAY); - return toIntExact(days); - } - - throw new IllegalArgumentException("Wrong type for temporal column: " + type); - } - - public static int getDayFromRange(ShardRange range) - { - Tuple min = range.getMinTuple(); - Tuple max = range.getMaxTuple(); - checkArgument(getOnlyElement(min.getTypes()).equals(getOnlyElement(max.getTypes())), "type of min and max is not same"); - - Type type = getOnlyElement(min.getTypes()); - if (type.equals(DATE)) { - return (int) getOnlyElement(min.getValues()); - } - - if (type.equals(TIMESTAMP_MILLIS)) { - long minValue = (long) getOnlyElement(min.getValues()); - long maxValue = (long) getOnlyElement(max.getValues()); - return determineDay(minValue, maxValue); - } - - throw new IllegalArgumentException("Wrong type for shard range: " + type); - } - - private static int determineDay(long rangeStart, long rangeEnd) - { - int startDay = toIntExact(Duration.ofMillis(rangeStart).toDays()); - int endDay = toIntExact(Duration.ofMillis(rangeEnd).toDays()); - if (startDay == endDay) { - return startDay; - } - - if ((endDay - startDay) > 1) { - // range spans multiple days, return the first full day - return startDay + 1; - } - - // range spans two days, return the day that has the larger time range - long millisInStartDay = Duration.ofDays(endDay).toMillis() - rangeStart; - long millisInEndDay = rangeEnd - Duration.ofDays(endDay).toMillis(); - return (millisInStartDay >= millisInEndDay) ? startDay : endDay; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/Tuple.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/Tuple.java deleted file mode 100644 index e1db17363b56..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/Tuple.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class Tuple - implements Comparable -{ - private final List types; - private final List values; - - public Tuple(Type type, Object value) - { - this(ImmutableList.of(type), ImmutableList.of(value)); - } - - public Tuple(List types, Object... values) - { - this(types, ImmutableList.copyOf(values)); - } - - public Tuple(List types, List values) - { - this.types = requireNonNull(types, "types is null"); - this.values = requireNonNull(values, "values is null"); - - checkArgument(!types.isEmpty(), "types is empty"); - checkArgument(types.size() == values.size(), "types and values must have the same number of elements"); - } - - public List getTypes() - { - return types; - } - - public List getValues() - { - return values; - } - - @Override - public int compareTo(Tuple o) - { - checkArgument(o.getTypes().size() == types.size(), "types must be of same size"); - for (int i = 0; i < types.size(); i++) { - checkArgument(o.getTypes().get(i).equals(types.get(i)), "types must be the same"); - } - - for (int i = 0; i < types.size(); i++) { - Object o1 = values.get(i); - Object o2 = o.getValues().get(i); - - int result = compare(o1, o2); - if (result != 0) { - return result; - } - } - return 0; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Tuple tuple = (Tuple) o; - return Objects.equals(types, tuple.types) && - Objects.equals(values, tuple.values); - } - - @Override - public int hashCode() - { - return Objects.hash(types, values); - } - - @SuppressWarnings("unchecked") - private static int compare(Object o1, Object o2) - { - return ((Comparable) o1).compareTo((T) o2); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("values", values) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ColumnRangesSystemTable.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ColumnRangesSystemTable.java deleted file mode 100644 index 6838c4201567..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ColumnRangesSystemTable.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.base.VerifyException; -import io.trino.plugin.raptor.legacy.RaptorTableHandle; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.FixedPageSource; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.JdbiException; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.joining; - -public class ColumnRangesSystemTable - implements SystemTable -{ - private static final String MIN_COLUMN_SUFFIX = "_min"; - private static final String MAX_COLUMN_SUFFIX = "_max"; - private static final String COLUMN_RANGES_TABLE_SUFFIX = "$column_ranges"; - - private final Jdbi dbi; - private final RaptorTableHandle sourceTable; - private final List indexedRaptorColumns; - private final ConnectorTableMetadata tableMetadata; - - public ColumnRangesSystemTable(RaptorTableHandle sourceTable, Jdbi dbi) - { - this.sourceTable = requireNonNull(sourceTable, "sourceTable is null"); - this.dbi = requireNonNull(dbi, "dbi is null"); - - this.indexedRaptorColumns = dbi.onDemand(MetadataDao.class) - .listTableColumns(sourceTable.getTableId()).stream() - .filter(column -> isIndexedType(column.getDataType())) - .collect(toImmutableList()); - List systemTableColumns = indexedRaptorColumns.stream() - .flatMap(column -> Stream.of( - new ColumnMetadata(column.getColumnName() + MIN_COLUMN_SUFFIX, column.getDataType()), - new ColumnMetadata(column.getColumnName() + MAX_COLUMN_SUFFIX, column.getDataType()))) - .collect(toImmutableList()); - SchemaTableName tableName = new SchemaTableName(sourceTable.getSchemaName(), sourceTable.getTableName() + COLUMN_RANGES_TABLE_SUFFIX); - this.tableMetadata = new ConnectorTableMetadata(tableName, systemTableColumns); - } - - public static Optional getSourceTable(SchemaTableName tableName) - { - if (tableName.getTableName().endsWith(COLUMN_RANGES_TABLE_SUFFIX) && - !tableName.getTableName().equals(COLUMN_RANGES_TABLE_SUFFIX)) { - int tableNameLength = tableName.getTableName().length() - COLUMN_RANGES_TABLE_SUFFIX.length(); - return Optional.of(new SchemaTableName( - tableName.getSchemaName(), - tableName.getTableName().substring(0, tableNameLength))); - } - return Optional.empty(); - } - - @Override - public Distribution getDistribution() - { - return SINGLE_COORDINATOR; - } - - @Override - public ConnectorTableMetadata getTableMetadata() - { - return tableMetadata; - } - - @Override - public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) - { - String metadataSqlQuery = getColumnRangesMetadataSqlQuery(sourceTable, indexedRaptorColumns); - List columnTypes = tableMetadata.getColumns().stream() - .map(ColumnMetadata::getType) - .collect(toImmutableList()); - - PageListBuilder pageListBuilder = new PageListBuilder(columnTypes); - - try (Connection connection = dbi.open().getConnection(); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(metadataSqlQuery)) { - if (resultSet.next()) { - pageListBuilder.beginRow(); - for (int i = 0; i < columnTypes.size(); ++i) { - BlockBuilder blockBuilder = pageListBuilder.nextBlockBuilder(); - Type columnType = columnTypes.get(i); - if (columnType.equals(BIGINT) || columnType.equals(DATE)) { - long value = resultSet.getLong(i + 1); - if (!resultSet.wasNull()) { - columnType.writeLong(blockBuilder, value); - } - else { - blockBuilder.appendNull(); - } - } - else if (columnType.equals(TIMESTAMP_MILLIS)) { - long value = resultSet.getLong(i + 1); - if (!resultSet.wasNull()) { - columnType.writeLong(blockBuilder, value * MICROSECONDS_PER_MILLISECOND); - } - else { - blockBuilder.appendNull(); - } - } - else if (columnType.equals(BOOLEAN)) { - boolean value = resultSet.getBoolean(i + 1); - if (!resultSet.wasNull()) { - BOOLEAN.writeBoolean(blockBuilder, value); - } - else { - blockBuilder.appendNull(); - } - } - else { - throw new VerifyException("Unknown or unsupported column type: " + columnType); - } - } - } - } - catch (SQLException | JdbiException e) { - throw metadataError(e); - } - - return new FixedPageSource(pageListBuilder.build()); - } - - private static boolean isIndexedType(Type type) - { - // We only consider the following types in the column_ranges system table - // Exclude INTEGER because we don't collect column stats for INTEGER type. - // Exclude DOUBLE because Java double is not completely compatible with MySQL double - // Exclude VARCHAR because they can be truncated - return type.equals(BOOLEAN) || type.equals(BIGINT) || type.equals(DATE) || type.equals(TIMESTAMP_MILLIS); - } - - private static String getColumnRangesMetadataSqlQuery(RaptorTableHandle raptorTableHandle, List raptorColumns) - { - String columns = raptorColumns.stream() - .flatMap(column -> Stream.of( - format("min(%s)", minColumn(column.getColumnId())), - format("max(%s)", maxColumn(column.getColumnId())))) - .collect(joining(", ")); - return format("SELECT %s FROM %s", columns, shardIndexTable(raptorTableHandle.getTableId())); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PageListBuilder.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PageListBuilder.java deleted file mode 100644 index 76800ff382fe..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PageListBuilder.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.Type; - -import java.util.List; - -class PageListBuilder -{ - private final PageBuilder pageBuilder; - private final ImmutableList.Builder pages = ImmutableList.builder(); - private int channel; - - public PageListBuilder(List types) - { - this.pageBuilder = new PageBuilder(types); - } - - public List build() - { - if (!pageBuilder.isEmpty()) { - pages.add(pageBuilder.build()); - pageBuilder.reset(); - } - return pages.build(); - } - - public void beginRow() - { - if (pageBuilder.isFull()) { - pages.add(pageBuilder.build()); - pageBuilder.reset(); - } - pageBuilder.declarePosition(); - channel = 0; - } - - public BlockBuilder nextBlockBuilder() - { - int currentChannel = channel; - channel++; - return pageBuilder.getBlockBuilder(currentChannel); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PreparedStatementBuilder.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PreparedStatementBuilder.java deleted file mode 100644 index 1230ef90bc1d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PreparedStatementBuilder.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Strings.isNullOrEmpty; -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.enableStreamingResults; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidToBytes; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static java.lang.String.format; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.sql.ResultSet.CONCUR_READ_ONLY; -import static java.sql.ResultSet.TYPE_FORWARD_ONLY; -import static java.util.Collections.nCopies; -import static java.util.UUID.fromString; - -public final class PreparedStatementBuilder -{ - private PreparedStatementBuilder() {} - - public static PreparedStatement create( - Connection connection, - String sql, - List columnNames, - List types, - Set uuidColumnIndexes, - TupleDomain tupleDomain) - throws SQLException - { - checkArgument(!isNullOrEmpty(sql), "sql is null or empty"); - - List bindValues = new ArrayList<>(256); - sql += getWhereClause(tupleDomain, columnNames, types, uuidColumnIndexes, bindValues); - - PreparedStatement statement = connection.prepareStatement(sql, TYPE_FORWARD_ONLY, CONCUR_READ_ONLY); - enableStreamingResults(statement); - - // bind values to statement - int bindIndex = 1; - for (ValueBuffer value : bindValues) { - bindField(value, statement, bindIndex, uuidColumnIndexes.contains(value.getColumnIndex())); - bindIndex++; - } - return statement; - } - - @SuppressWarnings("OptionalGetWithoutIsPresent") - private static String getWhereClause( - TupleDomain tupleDomain, - List columnNames, - List types, - Set uuidColumnIndexes, - List bindValues) - { - if (tupleDomain.isNone()) { - return ""; - } - - ImmutableList.Builder conjunctsBuilder = ImmutableList.builder(); - Map domainMap = tupleDomain.getDomains().get(); - for (Map.Entry entry : domainMap.entrySet()) { - int index = entry.getKey(); - String columnName = columnNames.get(index); - Type type = types.get(index); - conjunctsBuilder.add(toPredicate(index, columnName, type, entry.getValue(), uuidColumnIndexes, bindValues)); - } - List conjuncts = conjunctsBuilder.build(); - - if (conjuncts.isEmpty()) { - return ""; - } - StringBuilder where = new StringBuilder("WHERE "); - return Joiner.on(" AND\n").appendTo(where, conjuncts).toString(); - } - - private static String toPredicate( - int columnIndex, - String columnName, - Type type, - Domain domain, - Set uuidColumnIndexes, - List bindValues) - { - if (domain.getValues().isAll()) { - return domain.isNullAllowed() ? "TRUE" : columnName + " IS NOT NULL"; - } - if (domain.getValues().isNone()) { - return domain.isNullAllowed() ? columnName + " IS NULL" : "FALSE"; - } - - return domain.getValues().getValuesProcessor().transform( - ranges -> { - // Add disjuncts for ranges - List disjuncts = new ArrayList<>(); - List singleValues = new ArrayList<>(); - - // Add disjuncts for ranges - for (Range range : ranges.getOrderedRanges()) { - checkState(!range.isAll()); // Already checked - if (range.isSingleValue()) { - singleValues.add(range.getSingleValue()); - } - else { - List rangeConjuncts = new ArrayList<>(); - if (!range.isLowUnbounded()) { - Object bindValue = getBindValue(columnIndex, uuidColumnIndexes, range.getLowBoundedValue()); - rangeConjuncts.add(toBindPredicate(columnName, range.isLowInclusive() ? ">=" : ">")); - bindValues.add(ValueBuffer.create(columnIndex, type, bindValue)); - } - if (!range.isHighUnbounded()) { - Object bindValue = getBindValue(columnIndex, uuidColumnIndexes, range.getHighBoundedValue()); - rangeConjuncts.add(toBindPredicate(columnName, range.isHighInclusive() ? "<=" : "<")); - bindValues.add(ValueBuffer.create(columnIndex, type, bindValue)); - } - // If rangeConjuncts is null, then the range was ALL, which should already have been checked for - checkState(!rangeConjuncts.isEmpty()); - disjuncts.add("(" + Joiner.on(" AND ").join(rangeConjuncts) + ")"); - } - } - - // Add back all of the possible single values either as an equality or an IN predicate - if (singleValues.size() == 1) { - disjuncts.add(toBindPredicate(columnName, "=")); - bindValues.add(ValueBuffer.create(columnIndex, type, getBindValue(columnIndex, uuidColumnIndexes, getOnlyElement(singleValues)))); - } - else if (singleValues.size() > 1) { - disjuncts.add(columnName + " IN (" + Joiner.on(",").join(nCopies(singleValues.size(), "?")) + ")"); - for (Object singleValue : singleValues) { - bindValues.add(ValueBuffer.create(columnIndex, type, getBindValue(columnIndex, uuidColumnIndexes, singleValue))); - } - } - - // Add nullability disjuncts - checkState(!disjuncts.isEmpty()); - if (domain.isNullAllowed()) { - disjuncts.add(columnName + " IS NULL"); - } - - return "(" + Joiner.on(" OR ").join(disjuncts) + ")"; - }, - - discreteValues -> { - String values = Joiner.on(",").join(nCopies(discreteValues.getValues().size(), "?")); - String predicate = columnName + (discreteValues.isInclusive() ? "" : " NOT") + " IN (" + values + ")"; - for (Object value : discreteValues.getValues()) { - bindValues.add(ValueBuffer.create(columnIndex, type, getBindValue(columnIndex, uuidColumnIndexes, value))); - } - if (domain.isNullAllowed()) { - predicate = "(" + predicate + " OR " + columnName + " IS NULL)"; - } - return predicate; - }, - - allOrNone -> { - throw new IllegalStateException("Case should not be reachable"); - }); - } - - private static Object getBindValue(int columnIndex, Set uuidColumnIndexes, Object value) - { - if (uuidColumnIndexes.contains(columnIndex)) { - return uuidToBytes(fromString(((Slice) value).toStringUtf8())); - } - return value; - } - - private static String toBindPredicate(String columnName, String operator) - { - return format("%s %s ?", columnName, operator); - } - - private static void bindField(ValueBuffer valueBuffer, PreparedStatement preparedStatement, int parameterIndex, boolean isUuid) - throws SQLException - { - Type type = valueBuffer.getType(); - if (valueBuffer.isNull()) { - preparedStatement.setNull(parameterIndex, typeToSqlType(type)); - } - else if (type.getJavaType() == long.class) { - preparedStatement.setLong(parameterIndex, valueBuffer.getLong()); - } - else if (type.getJavaType() == double.class) { - preparedStatement.setDouble(parameterIndex, valueBuffer.getDouble()); - } - else if (type.getJavaType() == boolean.class) { - preparedStatement.setBoolean(parameterIndex, valueBuffer.getBoolean()); - } - else if (type.getJavaType() == Slice.class && isUuid) { - preparedStatement.setBytes(parameterIndex, valueBuffer.getSlice().getBytes()); - } - else if (type.getJavaType() == Slice.class) { - preparedStatement.setString(parameterIndex, new String(valueBuffer.getSlice().getBytes(), UTF_8)); - } - else { - throw new IllegalArgumentException("Unknown Java type: " + type.getJavaType()); - } - } - - private static int typeToSqlType(Type type) - { - if (type.equals(BIGINT)) { - return Types.BIGINT; - } - if (type.equals(DOUBLE)) { - return Types.DOUBLE; - } - if (type.equals(BOOLEAN)) { - return Types.BOOLEAN; - } - if (type instanceof VarcharType) { - return Types.VARCHAR; - } - if (type.equals(VARBINARY)) { - return Types.VARBINARY; - } - throw new IllegalArgumentException("Unknown type: " + type); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ResultSetValues.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ResultSetValues.java deleted file mode 100644 index 423d3c106f2a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ResultSetValues.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.base.VerifyException; -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.spi.type.Type; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; -import static io.airlift.slice.SizeOf.SIZE_OF_DOUBLE; -import static io.airlift.slice.SizeOf.SIZE_OF_LONG; -import static io.airlift.slice.Slices.wrappedBuffer; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static java.lang.String.format; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.requireNonNull; - -public class ResultSetValues -{ - private final boolean[] booleans; - private final long[] longs; - private final double[] doubles; - private final String[] strings; - private final boolean[] nulls; - private final List types; - - public ResultSetValues(List types) - { - this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); - - this.booleans = new boolean[types.size()]; - this.longs = new long[types.size()]; - this.doubles = new double[types.size()]; - this.strings = new String[types.size()]; - this.nulls = new boolean[types.size()]; - } - - int extractValues(ResultSet resultSet, Set uuidColumns, Set hexColumns) - throws SQLException - { - checkArgument(resultSet != null, "resultSet is null"); - int completedBytes = 0; - - for (int i = 0; i < types.size(); i++) { - Class javaType = types.get(i).getJavaType(); - - if (javaType == boolean.class) { - booleans[i] = resultSet.getBoolean(i + 1); - nulls[i] = resultSet.wasNull(); - if (!nulls[i]) { - completedBytes += SIZE_OF_BYTE; - } - } - else if (javaType == long.class) { - longs[i] = resultSet.getLong(i + 1); - nulls[i] = resultSet.wasNull(); - if (!nulls[i]) { - completedBytes += SIZE_OF_LONG; - } - } - else if (javaType == double.class) { - doubles[i] = resultSet.getDouble(i + 1); - nulls[i] = resultSet.wasNull(); - if (!nulls[i]) { - completedBytes += SIZE_OF_DOUBLE; - } - } - else if (javaType == Slice.class) { - if (uuidColumns.contains(i)) { - byte[] bytes = resultSet.getBytes(i + 1); - nulls[i] = resultSet.wasNull(); - strings[i] = nulls[i] ? null : uuidFromBytes(bytes).toString().toLowerCase(ENGLISH); - } - else if (hexColumns.contains(i)) { - long value = resultSet.getLong(i + 1); - nulls[i] = resultSet.wasNull(); - strings[i] = nulls[i] ? null : format("%016x", value); - } - else { - String value = resultSet.getString(i + 1); - nulls[i] = resultSet.wasNull(); - strings[i] = nulls[i] ? null : value; - } - - if (!nulls[i]) { - completedBytes += strings[i].length(); - } - } - else { - throw new VerifyException("Unknown Java type: " + javaType); - } - } - return completedBytes; - } - - public boolean getBoolean(int field) - { - return booleans[field]; - } - - public long getLong(int field) - { - return longs[field]; - } - - public double getDouble(int field) - { - return doubles[field]; - } - - public Slice getSlice(int field) - { - return wrappedBuffer(strings[field].getBytes(UTF_8)); - } - - public boolean isNull(int field) - { - return nulls[field]; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataRecordCursor.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataRecordCursor.java deleted file mode 100644 index 51e4650f53e0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataRecordCursor.java +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.JdbiException; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Iterator; -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkPositionIndex; -import static com.google.common.base.Preconditions.checkState; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_CORRUPT_METADATA; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static java.lang.String.format; -import static java.util.Collections.emptySet; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class ShardMetadataRecordCursor - implements RecordCursor -{ - private static final String SHARD_UUID = "shard_uuid"; - private static final String XXHASH64 = "xxhash64"; - private static final String SCHEMA_NAME = "table_schema"; - private static final String TABLE_NAME = "table_name"; - private static final String MIN_TIMESTAMP = "min_timestamp"; - private static final String MAX_TIMESTAMP = "max_timestamp"; - private static final String MIN_DATE = "min_date"; - private static final String MAX_DATE = "max_date"; - - public static final SchemaTableName SHARD_METADATA_TABLE_NAME = new SchemaTableName("system", "shards"); - public static final ConnectorTableMetadata SHARD_METADATA = new ConnectorTableMetadata( - SHARD_METADATA_TABLE_NAME, - ImmutableList.of( - new ColumnMetadata(SCHEMA_NAME, createUnboundedVarcharType()), - new ColumnMetadata(TABLE_NAME, createUnboundedVarcharType()), - new ColumnMetadata(SHARD_UUID, SHARD_UUID_COLUMN_TYPE), - new ColumnMetadata("bucket_number", BIGINT), - new ColumnMetadata("uncompressed_size", BIGINT), - new ColumnMetadata("compressed_size", BIGINT), - new ColumnMetadata("row_count", BIGINT), - new ColumnMetadata(XXHASH64, createVarcharType(16)), - new ColumnMetadata(MIN_TIMESTAMP, TIMESTAMP_MILLIS), - new ColumnMetadata(MAX_TIMESTAMP, TIMESTAMP_MILLIS), - new ColumnMetadata(MIN_DATE, DATE), - new ColumnMetadata(MAX_DATE, DATE))); - - private static final List COLUMNS = SHARD_METADATA.getColumns(); - private static final List TYPES = COLUMNS.stream().map(ColumnMetadata::getType).collect(toList()); - - private final Jdbi dbi; - private final MetadataDao metadataDao; - - private final Iterator tableIds; - private final List columnNames; - private final TupleDomain tupleDomain; - - private ResultSet resultSet; - private Connection connection; - private PreparedStatement statement; - private final ResultSetValues resultSetValues; - - private boolean closed; - private long completedBytes; - - public ShardMetadataRecordCursor(Jdbi dbi, TupleDomain tupleDomain) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.metadataDao = onDemandDao(dbi, MetadataDao.class); - this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); - this.tableIds = getTableIds(dbi, tupleDomain); - this.columnNames = createQualifiedColumnNames(); - this.resultSetValues = new ResultSetValues(TYPES); - this.resultSet = getNextResultSet(); - } - - private static String constructSqlTemplate(List columnNames, long tableId) - { - return format("SELECT %s\nFROM %s x\n" + - "JOIN shards ON (x.shard_id = shards.shard_id AND shards.table_id = %s)\n" + - "JOIN tables ON (tables.table_id = %s)\n", - Joiner.on(", ").join(columnNames), - shardIndexTable(tableId), - tableId, - tableId); - } - - private static List createQualifiedColumnNames() - { - return ImmutableList.builder() - .add("tables.schema_name") - .add("tables.table_name") - .add("shards." + COLUMNS.get(2).getName()) - .add("shards." + COLUMNS.get(3).getName()) - .add("shards." + COLUMNS.get(4).getName()) - .add("shards." + COLUMNS.get(5).getName()) - .add("shards." + COLUMNS.get(6).getName()) - .add("shards." + COLUMNS.get(7).getName()) - .add(MIN_TIMESTAMP) - .add(MAX_TIMESTAMP) - .add(MIN_DATE) - .add(MAX_DATE) - .build(); - } - - @Override - public long getCompletedBytes() - { - return completedBytes; - } - - @Override - public long getReadTimeNanos() - { - return 0; - } - - @Override - public Type getType(int field) - { - checkPositionIndex(field, TYPES.size()); - return TYPES.get(field); - } - - @Override - public boolean advanceNextPosition() - { - if (resultSet == null) { - close(); - } - - if (closed) { - return false; - } - - try { - while (!resultSet.next()) { - resultSet = getNextResultSet(); - if (resultSet == null) { - close(); - return false; - } - } - completedBytes += resultSetValues.extractValues( - resultSet, - ImmutableSet.of(getColumnIndex(SHARD_METADATA, SHARD_UUID)), - ImmutableSet.of(getColumnIndex(SHARD_METADATA, XXHASH64))); - return true; - } - catch (SQLException | JdbiException e) { - throw metadataError(e); - } - } - - @Override - public boolean getBoolean(int field) - { - checkFieldType(field, boolean.class); - return resultSetValues.getBoolean(field); - } - - @Override - public long getLong(int field) - { - checkFieldType(field, long.class); - return resultSetValues.getLong(field); - } - - @Override - public double getDouble(int field) - { - checkFieldType(field, double.class); - return resultSetValues.getDouble(field); - } - - @Override - public Slice getSlice(int field) - { - checkFieldType(field, Slice.class); - return resultSetValues.getSlice(field); - } - - @Override - public Object getObject(int field) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isNull(int field) - { - checkState(!closed, "cursor is closed"); - checkPositionIndex(field, TYPES.size()); - return resultSetValues.isNull(field); - } - - @Override - public void close() - { - closed = true; - closeCurrentResultSet(); - } - - @SuppressWarnings("unused") - private void closeCurrentResultSet() - { - // use try-with-resources to close everything properly - //noinspection EmptyTryBlock - try (Connection connection = this.connection; - Statement statement = this.statement; - ResultSet resultSet = this.resultSet) { - // do nothing - } - catch (SQLException _) { - } - } - - private ResultSet getNextResultSet() - { - closeCurrentResultSet(); - - if (!tableIds.hasNext()) { - return null; - } - - Long tableId = tableIds.next(); - Long columnId = metadataDao.getTemporalColumnId(tableId); - List columnNames; - - if (columnId == null) { - columnNames = getMappedColumnNames("null", "null", "null", "null"); - } - else { - Type temporalType = metadataDao.getTableColumn(tableId, columnId).getDataType(); - if (temporalType.equals(DATE)) { - columnNames = getMappedColumnNames("null", "null", minColumn(columnId), maxColumn(columnId)); - } - else if (temporalType.equals(TIMESTAMP_MILLIS)) { - columnNames = getMappedColumnNames(minColumn(columnId), maxColumn(columnId), "null", "null"); - } - else { - throw new TrinoException(RAPTOR_CORRUPT_METADATA, "Temporal column should be of type date or timestamp, not " + temporalType.getDisplayName()); - } - } - - try { - connection = dbi.open().getConnection(); - statement = PreparedStatementBuilder.create( - connection, - constructSqlTemplate(columnNames, tableId), - columnNames, - TYPES, - ImmutableSet.of(getColumnIndex(SHARD_METADATA, SHARD_UUID)), - tupleDomain); - return statement.executeQuery(); - } - catch (SQLException | JdbiException e) { - close(); - throw metadataError(e); - } - } - - private List getMappedColumnNames(String minTimestampColumn, String maxTimestampColumn, String minDateColumn, String maxDateColumn) - { - ImmutableList.Builder builder = ImmutableList.builder(); - for (String column : columnNames) { - switch (column) { - case MIN_TIMESTAMP: - builder.add(minTimestampColumn); - break; - case MAX_TIMESTAMP: - builder.add(maxTimestampColumn); - break; - case MIN_DATE: - builder.add(minDateColumn); - break; - case MAX_DATE: - builder.add(maxDateColumn); - break; - default: - builder.add(column); - break; - } - } - return builder.build(); - } - - @VisibleForTesting - static Iterator getTableIds(Jdbi dbi, TupleDomain tupleDomain) - { - ImmutableList.Builder tableIds = ImmutableList.builder(); - try (Connection connection = dbi.open().getConnection(); - PreparedStatement statement = PreparedStatementBuilder.create( - connection, - "SELECT table_id FROM tables ", - List.of("schema_name", "table_name"), - List.of(VARCHAR, VARCHAR), - emptySet(), - tupleDomain.filter((key, domain) -> (key == 0) || (key == 1))); - ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - tableIds.add(resultSet.getLong("table_id")); - } - } - catch (SQLException | JdbiException e) { - throw metadataError(e); - } - return tableIds.build().iterator(); - } - - private static int getColumnIndex(ConnectorTableMetadata tableMetadata, String columnName) - { - List columns = tableMetadata.getColumns(); - for (int i = 0; i < columns.size(); i++) { - if (columns.get(i).getName().equals(columnName)) { - return i; - } - } - throw new IllegalArgumentException(format("Column '%s' not found", columnName)); - } - - private void checkFieldType(int field, Class clazz) - { - checkState(!closed, "cursor is closed"); - Type type = getType(field); - checkArgument(type.getJavaType() == clazz, "Type %s cannot be read as %s", type, clazz.getSimpleName()); - } - - private static String getStringValue(Object value) - { - return ((Slice) value).toStringUtf8(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataSystemTable.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataSystemTable.java deleted file mode 100644 index b4e7a8364746..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataSystemTable.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.predicate.TupleDomain; -import org.jdbi.v3.core.Jdbi; - -import static io.trino.plugin.raptor.legacy.systemtables.ShardMetadataRecordCursor.SHARD_METADATA; -import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; -import static java.util.Objects.requireNonNull; - -public class ShardMetadataSystemTable - implements SystemTable -{ - private final Jdbi dbi; - - @Inject - public ShardMetadataSystemTable(@ForMetadata Jdbi dbi) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - } - - @Override - public Distribution getDistribution() - { - return SINGLE_COORDINATOR; - } - - @Override - public ConnectorTableMetadata getTableMetadata() - { - return SHARD_METADATA; - } - - @Override - public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) - { - return new ShardMetadataRecordCursor(dbi, constraint); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java deleted file mode 100644 index 28192417bca6..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.PeekingIterator; -import com.google.inject.Inject; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.metadata.ColumnMetadataRow; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.TableMetadataRow; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.block.ArrayBlockBuilder; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.FixedPageSource; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.predicate.NullableValue; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.TypeManager; -import org.jdbi.v3.core.Jdbi; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.SortedMap; -import java.util.TreeMap; - -import static com.google.common.collect.Iterators.peekingIterator; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_CORRUPT_METADATA; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; -import static io.trino.spi.predicate.TupleDomain.extractFixedValues; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class TableMetadataSystemTable - implements SystemTable -{ - private static final String TABLE_NAME = "table_name"; - private static final String SCHEMA_NAME = "table_schema"; - - private final MetadataDao dao; - private final ConnectorTableMetadata tableMetadata; - - @Inject - public TableMetadataSystemTable(@ForMetadata Jdbi dbi, TypeManager typeManager) - { - this.dao = onDemandDao(dbi, MetadataDao.class); - requireNonNull(typeManager, "typeManager is null"); - - this.tableMetadata = new ConnectorTableMetadata( - new SchemaTableName("system", "tables"), - ImmutableList.of( - new ColumnMetadata(SCHEMA_NAME, VARCHAR), - new ColumnMetadata(TABLE_NAME, VARCHAR), - new ColumnMetadata("temporal_column", VARCHAR), - new ColumnMetadata("ordering_columns", new ArrayType(VARCHAR)), - new ColumnMetadata("distribution_name", VARCHAR), - new ColumnMetadata("bucket_count", BIGINT), - new ColumnMetadata("bucketing_columns", new ArrayType(VARCHAR)), - new ColumnMetadata("organized", BOOLEAN))); - } - - @Override - public Distribution getDistribution() - { - return SINGLE_COORDINATOR; - } - - @Override - public ConnectorTableMetadata getTableMetadata() - { - return tableMetadata; - } - - @Override - public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) - { - return new FixedPageSource(buildPages(dao, tableMetadata, constraint)); - } - - private static List buildPages(MetadataDao dao, ConnectorTableMetadata tableMetadata, TupleDomain tupleDomain) - { - Map domainValues = extractFixedValues(tupleDomain).orElse(ImmutableMap.of()); - String schemaName = getStringValue(domainValues.get(getColumnIndex(tableMetadata, SCHEMA_NAME))); - String tableName = getStringValue(domainValues.get(getColumnIndex(tableMetadata, TABLE_NAME))); - - PageListBuilder pageBuilder = new PageListBuilder(tableMetadata.getColumns().stream() - .map(ColumnMetadata::getType) - .collect(toList())); - - List tableRows = dao.getTableMetadataRows(schemaName, tableName); - PeekingIterator columnRowIterator = peekingIterator(dao.getColumnMetadataRows(schemaName, tableName).iterator()); - - for (TableMetadataRow tableRow : tableRows) { - while (columnRowIterator.hasNext() && columnRowIterator.peek().getTableId() < tableRow.getTableId()) { - columnRowIterator.next(); - } - - String temporalColumnName = null; - SortedMap sortColumnNames = new TreeMap<>(); - SortedMap bucketColumnNames = new TreeMap<>(); - OptionalLong temporalColumnId = tableRow.getTemporalColumnId(); - while (columnRowIterator.hasNext() && columnRowIterator.peek().getTableId() == tableRow.getTableId()) { - ColumnMetadataRow columnRow = columnRowIterator.next(); - if (temporalColumnId.isPresent() && columnRow.getColumnId() == temporalColumnId.getAsLong()) { - temporalColumnName = columnRow.getColumnName(); - } - OptionalInt sortOrdinalPosition = columnRow.getSortOrdinalPosition(); - if (sortOrdinalPosition.isPresent()) { - sortColumnNames.put(sortOrdinalPosition.getAsInt(), columnRow.getColumnName()); - } - OptionalInt bucketOrdinalPosition = columnRow.getBucketOrdinalPosition(); - if (bucketOrdinalPosition.isPresent()) { - bucketColumnNames.put(bucketOrdinalPosition.getAsInt(), columnRow.getColumnName()); - } - } - - pageBuilder.beginRow(); - - // schema_name, table_name - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(tableRow.getSchemaName())); - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(tableRow.getTableName())); - - // temporal_column - if (temporalColumnId.isPresent()) { - if (temporalColumnName == null) { - throw new TrinoException(RAPTOR_CORRUPT_METADATA, format("Table ID %s has corrupt metadata (invalid temporal column ID)", tableRow.getTableId())); - } - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(temporalColumnName)); - } - else { - pageBuilder.nextBlockBuilder().appendNull(); - } - - // ordering_columns - writeArray(pageBuilder.nextBlockBuilder(), sortColumnNames.values()); - - // distribution_name - Optional distributionName = tableRow.getDistributionName(); - if (distributionName.isPresent()) { - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(distributionName.get())); - } - else { - pageBuilder.nextBlockBuilder().appendNull(); - } - - // bucket_count - OptionalInt bucketCount = tableRow.getBucketCount(); - if (bucketCount.isPresent()) { - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), bucketCount.getAsInt()); - } - else { - pageBuilder.nextBlockBuilder().appendNull(); - } - - // bucketing_columns - writeArray(pageBuilder.nextBlockBuilder(), bucketColumnNames.values()); - - // organized - BOOLEAN.writeBoolean(pageBuilder.nextBlockBuilder(), tableRow.isOrganized()); - } - - return pageBuilder.build(); - } - - private static void writeArray(BlockBuilder blockBuilder, Collection values) - { - if (values.isEmpty()) { - blockBuilder.appendNull(); - } - else { - ((ArrayBlockBuilder) blockBuilder).buildEntry(elementBuilder -> { - for (String value : values) { - VARCHAR.writeSlice(elementBuilder, utf8Slice(value)); - } - }); - } - } - - static int getColumnIndex(ConnectorTableMetadata tableMetadata, String columnName) - { - List columns = tableMetadata.getColumns(); - for (int i = 0; i < columns.size(); i++) { - if (columns.get(i).getName().equals(columnName)) { - return i; - } - } - throw new IllegalArgumentException(format("Column '%s' not found", columnName)); - } - - static String getStringValue(NullableValue value) - { - if ((value == null) || value.isNull()) { - return null; - } - return ((Slice) value.getValue()).toStringUtf8(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableStatsSystemTable.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableStatsSystemTable.java deleted file mode 100644 index bde918a3781b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableStatsSystemTable.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.TableStatsRow; -import io.trino.spi.Page; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.FixedPageSource; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.predicate.NullableValue; -import io.trino.spi.predicate.TupleDomain; -import org.jdbi.v3.core.Jdbi; - -import java.util.List; -import java.util.Map; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.systemtables.TableMetadataSystemTable.getColumnIndex; -import static io.trino.plugin.raptor.legacy.systemtables.TableMetadataSystemTable.getStringValue; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; -import static io.trino.spi.predicate.TupleDomain.extractFixedValues; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static java.util.stream.Collectors.toList; - -public class TableStatsSystemTable - implements SystemTable -{ - private static final String SCHEMA_NAME = "table_schema"; - private static final String TABLE_NAME = "table_name"; - - private static final ConnectorTableMetadata METADATA = new ConnectorTableMetadata( - new SchemaTableName("system", "table_stats"), - ImmutableList.builder() - .add(new ColumnMetadata(SCHEMA_NAME, createUnboundedVarcharType())) - .add(new ColumnMetadata(TABLE_NAME, createUnboundedVarcharType())) - .add(new ColumnMetadata("create_time", TIMESTAMP_MILLIS)) - .add(new ColumnMetadata("update_time", TIMESTAMP_MILLIS)) - .add(new ColumnMetadata("table_version", BIGINT)) - .add(new ColumnMetadata("shard_count", BIGINT)) - .add(new ColumnMetadata("row_count", BIGINT)) - .add(new ColumnMetadata("compressed_size", BIGINT)) - .add(new ColumnMetadata("uncompressed_size", BIGINT)) - .build()); - - private final MetadataDao dao; - - @Inject - public TableStatsSystemTable(@ForMetadata Jdbi dbi) - { - this.dao = onDemandDao(dbi, MetadataDao.class); - } - - @Override - public Distribution getDistribution() - { - return SINGLE_COORDINATOR; - } - - @Override - public ConnectorTableMetadata getTableMetadata() - { - return METADATA; - } - - @Override - public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) - { - return new FixedPageSource(buildPages(dao, constraint)); - } - - private static List buildPages(MetadataDao dao, TupleDomain tupleDomain) - { - Map domainValues = extractFixedValues(tupleDomain).orElse(ImmutableMap.of()); - String schemaName = getStringValue(domainValues.get(getColumnIndex(METADATA, SCHEMA_NAME))); - String tableName = getStringValue(domainValues.get(getColumnIndex(METADATA, TABLE_NAME))); - - PageListBuilder pageBuilder = new PageListBuilder(METADATA.getColumns().stream() - .map(ColumnMetadata::getType) - .collect(toList())); - - for (TableStatsRow row : dao.getTableStatsRows(schemaName, tableName)) { - pageBuilder.beginRow(); - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(row.getSchemaName())); - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(row.getTableName())); - TIMESTAMP_MILLIS.writeLong(pageBuilder.nextBlockBuilder(), row.getCreateTime() * MICROSECONDS_PER_MILLISECOND); - TIMESTAMP_MILLIS.writeLong(pageBuilder.nextBlockBuilder(), row.getUpdateTime() * MICROSECONDS_PER_MILLISECOND); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getTableVersion()); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getShardCount()); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getRowCount()); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getCompressedSize()); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getUncompressedSize()); - } - - return pageBuilder.build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ValueBuffer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ValueBuffer.java deleted file mode 100644 index 97f7bdf56c9b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ValueBuffer.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.primitives.Primitives; -import io.airlift.slice.Slice; -import io.trino.spi.type.Type; -import jakarta.annotation.Nullable; - -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.slice.Slices.wrappedBuffer; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public class ValueBuffer -{ - private final int columnIndex; - private final Type type; - private final Object value; - - private ValueBuffer(int columnIndex, Type type, @Nullable Object value) - { - checkArgument(columnIndex >= 0, "columnIndex must be > 0"); - this.columnIndex = columnIndex; - this.type = requireNonNull(type, "type is null"); - this.value = value; - } - - public static ValueBuffer create(int columnIndex, Type type, @Nullable Object value) - { - if (value == null) { - return new ValueBuffer(columnIndex, type, null); - } - return new ValueBuffer(columnIndex, type, normalizeValue(type, value)); - } - - private static Object normalizeValue(Type type, @Nullable Object value) - { - if (value == null) { - return value; - } - - Class javaType = type.getJavaType(); - if (javaType.isPrimitive()) { - checkArgument(Primitives.wrap(javaType).isInstance(value), "Type %s incompatible with %s", type, value); - return value; - } - if (javaType == Slice.class) { - if (value instanceof Slice) { - return value; - } - if (value instanceof String) { - return utf8Slice(((String) value)); - } - if (value instanceof byte[]) { - return wrappedBuffer((byte[]) value); - } - } - throw new IllegalArgumentException(format("Type %s incompatible with %s", type, value)); - } - - @Nullable - public Object getValue() - { - checkState(!isNull()); - return value; - } - - public int getColumnIndex() - { - return columnIndex; - } - - public long getLong() - { - checkState(!isNull()); - checkArgument(type.getJavaType() == long.class, "Type %s cannot be read as long", type); - return (Long) value; - } - - public double getDouble() - { - checkState(!isNull()); - checkArgument(type.getJavaType() == double.class, "Type %s cannot be read as double", type); - return (Double) value; - } - - public boolean getBoolean() - { - checkState(!isNull()); - checkArgument(type.getJavaType() == boolean.class, "Type %s cannot be read as boolean", type); - return (Boolean) value; - } - - public Slice getSlice() - { - checkState(!isNull()); - checkArgument(type.getJavaType() == Slice.class, "Type %s cannot be read as Slice", type); - return (Slice) value; - } - - public Type getType() - { - return type; - } - - public boolean isNull() - { - return value == null; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ValueBuffer that = (ValueBuffer) o; - return columnIndex == that.columnIndex && - Objects.equals(type, that.type) && - Objects.equals(value, that.value); - } - - @Override - public int hashCode() - { - return Objects.hash(columnIndex, type, value); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("columnIndex", columnIndex) - .add("type", type) - .add("value", value) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ArrayUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ArrayUtil.java deleted file mode 100644 index c5c1ad35b0e0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ArrayUtil.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.collect.ImmutableList; - -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.List; - -public final class ArrayUtil -{ - private ArrayUtil() {} - - /** - * Unpack an array of big endian ints. - */ - public static List intArrayFromBytes(byte[] bytes) - { - ImmutableList.Builder list = ImmutableList.builder(); - ByteBuffer buffer = ByteBuffer.wrap(bytes); - while (buffer.hasRemaining()) { - list.add(buffer.getInt()); - } - return list.build(); - } - - /** - * Pack an array of ints as big endian. - */ - public static byte[] intArrayToBytes(Collection values) - { - ByteBuffer buffer = ByteBuffer.allocate(values.size() * Integer.BYTES); - for (int value : values) { - buffer.putInt(value); - } - return buffer.array(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/Closer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/Closer.java deleted file mode 100644 index 4a79a1a144d8..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/Closer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import static java.util.Objects.requireNonNull; - -public class Closer - implements AutoCloseable -{ - private final T delegate; - private final Cleaner cleaner; - - public static Closer closer(T delegate, Cleaner cleaner) - { - return new Closer<>(delegate, cleaner); - } - - private Closer(T delegate, Cleaner cleaner) - { - this.delegate = requireNonNull(delegate, "delegate is null"); - this.cleaner = requireNonNull(cleaner, "cleaner is null"); - } - - public T get() - { - return delegate; - } - - @Override - public void close() - throws X - { - cleaner.close(delegate); - } - - public interface Cleaner - { - void close(T object) - throws X; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java deleted file mode 100644 index 3d346df637ef..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import io.trino.spi.Page; -import io.trino.spi.connector.ConnectorPageSource; - -import java.io.IOException; -import java.util.Iterator; - -import static java.util.Objects.requireNonNull; - -public class ConcatPageSource - implements ConnectorPageSource -{ - private final Iterator iterator; - - private ConnectorPageSource current; - private long completedBytes; - private long readTimeNanos; - - public ConcatPageSource(Iterator iterator) - { - this.iterator = requireNonNull(iterator, "iterator is null"); - } - - @Override - public long getCompletedBytes() - { - setup(); - return completedBytes + ((current != null) ? current.getCompletedBytes() : 0); - } - - @Override - public long getReadTimeNanos() - { - setup(); - return readTimeNanos + ((current != null) ? current.getReadTimeNanos() : 0); - } - - @Override - public boolean isFinished() - { - setup(); - return current == null; - } - - @Override - public Page getNextPage() - { - while (true) { - setup(); - - if (current == null) { - return null; - } - if (!current.isFinished()) { - return current.getNextPage(); - } - - completedBytes += current.getCompletedBytes(); - readTimeNanos += current.getReadTimeNanos(); - current = null; - } - } - - @Override - public long getMemoryUsage() - { - return (current != null) ? current.getMemoryUsage() : 0; - } - - @Override - public void close() - throws IOException - { - if (current != null) { - current.close(); - } - } - - private void setup() - { - if ((current == null) && iterator.hasNext()) { - current = iterator.next(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DaoSupplier.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DaoSupplier.java deleted file mode 100644 index 7bfed77b2685..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DaoSupplier.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; - -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static java.util.Objects.requireNonNull; - -public class DaoSupplier -{ - private final Class type; - private final T dao; - - public DaoSupplier(Jdbi dbi, Class type) - { - requireNonNull(dbi, "dbi is null"); - requireNonNull(type, "type is null"); - this.type = type; - this.dao = onDemandDao(dbi, type); - } - - public T onDemand() - { - return dao; - } - - public T attach(Handle handle) - { - return handle.attach(type); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DatabaseUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DatabaseUtil.java deleted file mode 100644 index 3e6604bfb289..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DatabaseUtil.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.base.Throwables; -import com.mysql.cj.jdbc.JdbcStatement; -import io.trino.spi.TrinoException; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.HandleCallback; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.JdbiException; - -import java.lang.reflect.InvocationTargetException; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.function.Consumer; -import java.util.function.Predicate; - -import static com.google.common.base.Throwables.throwIfInstanceOf; -import static com.google.common.reflect.Reflection.newProxy; -import static com.mysql.cj.exceptions.MysqlErrorNumbers.ER_TRANS_CACHE_FULL; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_METADATA_ERROR; -import static java.sql.Types.INTEGER; -import static java.util.Objects.requireNonNull; - -public final class DatabaseUtil -{ - private DatabaseUtil() {} - - public static T onDemandDao(Jdbi dbi, Class daoType) - { - requireNonNull(dbi, "dbi is null"); - return newProxy(daoType, (proxy, method, args) -> { - try (Handle handle = dbi.open()) { - T dao = handle.attach(daoType); - return method.invoke(dao, args); - } - catch (JdbiException e) { - throw metadataError(e); - } - catch (InvocationTargetException e) { - throw metadataError(e.getCause()); - } - }); - } - - public static T runTransaction(Jdbi dbi, HandleCallback callback) - { - try { - return dbi.inTransaction(callback); - } - catch (JdbiException e) { - if (e.getCause() != null) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - } - throw metadataError(e); - } - } - - public static void daoTransaction(Jdbi dbi, Class daoType, Consumer callback) - { - runTransaction(dbi, handle -> { - callback.accept(handle.attach(daoType)); - return null; - }); - } - - public static TrinoException metadataError(Throwable cause, String message) - { - return new TrinoException(RAPTOR_METADATA_ERROR, message, cause); - } - - public static TrinoException metadataError(Throwable cause) - { - return metadataError(cause, "Failed to perform metadata operation"); - } - - /** - * Run an SQL query as ignoring any constraint violations. - * This allows idempotent inserts (equivalent to INSERT IGNORE). - */ - public static void runIgnoringConstraintViolation(Runnable task) - { - try { - task.run(); - } - catch (RuntimeException e) { - if (!sqlCodeStartsWith(e, "23")) { - throw e; - } - } - } - - public static void enableStreamingResults(Statement statement) - throws SQLException - { - if (statement.isWrapperFor(JdbcStatement.class)) { - statement.unwrap(JdbcStatement.class).enableStreamingResults(); - } - } - - public static OptionalInt getOptionalInt(ResultSet rs, String name) - throws SQLException - { - int value = rs.getInt(name); - return rs.wasNull() ? OptionalInt.empty() : OptionalInt.of(value); - } - - public static OptionalLong getOptionalLong(ResultSet rs, String name) - throws SQLException - { - long value = rs.getLong(name); - return rs.wasNull() ? OptionalLong.empty() : OptionalLong.of(value); - } - - public static Long getBoxedLong(ResultSet rs, String name) - throws SQLException - { - long value = rs.getLong(name); - return rs.wasNull() ? null : value; - } - - public static void bindOptionalInt(PreparedStatement statement, int index, OptionalInt value) - throws SQLException - { - if (value.isPresent()) { - statement.setInt(index, value.getAsInt()); - } - else { - statement.setNull(index, INTEGER); - } - } - - public static boolean isSyntaxOrAccessError(Exception e) - { - return sqlCodeStartsWith(e, "42"); - } - - public static boolean isTransactionCacheFullError(Exception e) - { - return mySqlErrorCodeMatches(e, ER_TRANS_CACHE_FULL); - } - - /** - * Check if an exception is caused by a MySQL exception of certain error code - */ - private static boolean mySqlErrorCodeMatches(Exception e, int errorCode) - { - return Throwables.getCausalChain(e).stream() - .filter(SQLException.class::isInstance) - .map(SQLException.class::cast) - .filter(t -> t.getErrorCode() == errorCode) - .map(Throwable::getStackTrace) - .anyMatch(isMySQLException()); - } - - private static Predicate isMySQLException() - { - // check if the exception is a mysql exception by matching the package name in the stack trace - return s -> Arrays.stream(s) - .map(StackTraceElement::getClassName) - .anyMatch(t -> t.startsWith("com.mysql.jdbc.")); - } - - private static boolean sqlCodeStartsWith(Exception e, String code) - { - for (Throwable throwable : Throwables.getCausalChain(e)) { - if (throwable instanceof SQLException) { - String state = ((SQLException) throwable).getSQLState(); - if (state != null && state.startsWith(code)) { - return true; - } - } - } - return false; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/MetadataUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/MetadataUtil.java deleted file mode 100644 index 96262838bf5a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/MetadataUtil.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.util.Locale.ENGLISH; - -public final class MetadataUtil -{ - private MetadataUtil() {} - - public static String checkSchemaName(String schemaName) - { - return checkLowerCase(schemaName, "schemaName"); - } - - public static String checkTableName(String tableName) - { - return checkLowerCase(tableName, "tableName"); - } - - private static String checkLowerCase(String value, String name) - { - if (value == null) { - throw new NullPointerException(format("%s is null", name)); - } - checkArgument(value.equals(value.toLowerCase(ENGLISH)), "%s is not lowercase", name); - return value; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PageBuffer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PageBuffer.java deleted file mode 100644 index 5a9d60e3872d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PageBuffer.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.collect.ImmutableList; -import io.trino.plugin.raptor.legacy.storage.StoragePageSink; -import io.trino.spi.Page; -import io.trino.spi.PageSorter; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.type.Type; - -import java.util.ArrayList; -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.Math.toIntExact; -import static java.util.Objects.requireNonNull; - -public class PageBuffer -{ - private final long maxMemoryBytes; - private final StoragePageSink storagePageSink; - private final List columnTypes; - private final List sortFields; - private final List sortOrders; - private final PageSorter pageSorter; - private final List pages = new ArrayList<>(); - - private long usedMemoryBytes; - private long rowCount; - - public PageBuffer( - long maxMemoryBytes, - StoragePageSink storagePageSink, - List columnTypes, - List sortFields, - List sortOrders, - PageSorter pageSorter) - { - checkArgument(maxMemoryBytes > 0, "maxMemoryBytes must be positive"); - this.maxMemoryBytes = maxMemoryBytes; - this.columnTypes = requireNonNull(columnTypes, "columnTypes is null"); - this.sortFields = ImmutableList.copyOf(requireNonNull(sortFields, "sortFields is null")); - this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); - this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); - this.storagePageSink = requireNonNull(storagePageSink, "storagePageSink is null"); - } - - public StoragePageSink getStoragePageSink() - { - return storagePageSink; - } - - public long getUsedMemoryBytes() - { - return usedMemoryBytes; - } - - public void add(Page page) - { - flushIfNecessary(page.getPositionCount()); - pages.add(page); - usedMemoryBytes += page.getSizeInBytes(); - rowCount += page.getPositionCount(); - } - - public void flush() - { - if (pages.isEmpty()) { - return; - } - - if (sortFields.isEmpty()) { - storagePageSink.appendPages(pages); - } - else { - appendSorted(); - } - - storagePageSink.flush(); - - pages.clear(); - rowCount = 0; - usedMemoryBytes = 0; - } - - private void appendSorted() - { - long[] addresses = pageSorter.sort(columnTypes, pages, sortFields, sortOrders, toIntExact(rowCount)); - - int[] pageIndex = new int[addresses.length]; - int[] positionIndex = new int[addresses.length]; - for (int i = 0; i < addresses.length; i++) { - pageIndex[i] = pageSorter.decodePageIndex(addresses[i]); - positionIndex[i] = pageSorter.decodePositionIndex(addresses[i]); - } - - storagePageSink.appendPages(pages, pageIndex, positionIndex); - } - - private void flushIfNecessary(int rowsToAdd) - { - if (storagePageSink.isFull() || !canAddRows(rowsToAdd)) { - flush(); - } - } - - private boolean canAddRows(int rowsToAdd) - { - return (usedMemoryBytes < maxMemoryBytes) && - ((rowCount + rowsToAdd) < Integer.MAX_VALUE); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PrioritizedFifoExecutor.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PrioritizedFifoExecutor.java deleted file mode 100644 index a4094be27522..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PrioritizedFifoExecutor.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.collect.ComparisonChain; -import com.google.common.util.concurrent.ExecutionList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.errorprone.annotations.ThreadSafe; -import io.airlift.log.Logger; - -import java.util.Comparator; -import java.util.Objects; -import java.util.Queue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.FutureTask; -import java.util.concurrent.PriorityBlockingQueue; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -/** - * This class is based on io.airlift.concurrent.BoundedExecutor - */ -@ThreadSafe -public class PrioritizedFifoExecutor -{ - private static final Logger log = Logger.get(PrioritizedFifoExecutor.class); - - private final Queue> queue; - private final AtomicInteger queueSize = new AtomicInteger(0); - private final AtomicLong sequenceNumber = new AtomicLong(0); - private final Runnable triggerTask = this::executeOrMerge; - - private final ExecutorService executorService; - private final int maxThreads; - private final Comparator taskComparator; - - public PrioritizedFifoExecutor(ExecutorService coreExecutor, int maxThreads, Comparator taskComparator) - { - checkArgument(maxThreads > 0, "maxThreads must be greater than zero"); - - this.taskComparator = requireNonNull(taskComparator, "taskComparator is null"); - this.executorService = requireNonNull(coreExecutor, "coreExecutor is null"); - this.maxThreads = maxThreads; - this.queue = new PriorityBlockingQueue<>(maxThreads); - } - - public ListenableFuture submit(T task) - { - FifoRunnableTask fifoTask = new FifoRunnableTask<>(task, sequenceNumber.incrementAndGet(), taskComparator); - queue.add(fifoTask); - executorService.submit(triggerTask); - return fifoTask; - } - - private void executeOrMerge() - { - int size = queueSize.incrementAndGet(); - if (size > maxThreads) { - return; - } - do { - try { - queue.poll().run(); - } - catch (Throwable e) { - log.error(e, "Task failed"); - } - } - while (queueSize.getAndDecrement() > maxThreads); - } - - private static class FifoRunnableTask - extends FutureTask - implements ListenableFuture, Comparable> - { - private final ExecutionList executionList = new ExecutionList(); - private final T task; - private final long sequenceNumber; - private final Comparator taskComparator; - - public FifoRunnableTask(T task, long sequenceNumber, Comparator taskComparator) - { - super(requireNonNull(task, "task is null"), null); - this.task = task; - this.sequenceNumber = sequenceNumber; - this.taskComparator = requireNonNull(taskComparator, "taskComparator is null"); - } - - @Override - public void addListener(Runnable listener, Executor executor) - { - executionList.add(listener, executor); - } - - @Override - protected void done() - { - executionList.execute(); - } - - @Override - public int compareTo(FifoRunnableTask other) - { - return ComparisonChain.start() - .compare(this.task, other.task, taskComparator) - .compare(this.sequenceNumber, other.sequenceNumber) - .result(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - FifoRunnableTask other = (FifoRunnableTask) o; - return Objects.equals(this.task, other.task) && - this.sequenceNumber == other.sequenceNumber; - } - - @Override - public int hashCode() - { - return Objects.hash(task, sequenceNumber); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SynchronizedResultIterator.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SynchronizedResultIterator.java deleted file mode 100644 index 2562cb727761..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SynchronizedResultIterator.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.errorprone.annotations.concurrent.GuardedBy; -import org.jdbi.v3.core.result.ResultIterator; -import org.jdbi.v3.core.statement.StatementContext; - -import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; - -public class SynchronizedResultIterator - implements ResultIterator -{ - @GuardedBy("this") - private final ResultIterator iterator; - - @GuardedBy("this") - private boolean closed; - - public SynchronizedResultIterator(ResultIterator iterator) - { - this.iterator = requireNonNull(iterator, "iterator is null"); - } - - @Override - public synchronized boolean hasNext() - { - checkState(!closed, "already closed"); - return iterator.hasNext(); - } - - @Override - public synchronized T next() - { - checkState(!closed, "already closed"); - return iterator.next(); - } - - @Override - public synchronized void close() - { - if (!closed) { - closed = true; - iterator.close(); - } - } - - @Override - public synchronized StatementContext getContext() - { - return iterator.getContext(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SyncingFileOutputStream.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SyncingFileOutputStream.java deleted file mode 100644 index c952b0bb682e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SyncingFileOutputStream.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.io.Closer; -import io.airlift.slice.XxHash64; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; - -public class SyncingFileOutputStream - extends FileOutputStream -{ - private final byte[] oneByte = new byte[1]; - private final XxHash64 hash = new XxHash64(); - private final File file; - private boolean closed; - - public SyncingFileOutputStream(File file) - throws FileNotFoundException - { - super(file); - this.file = file; - } - - @Override - public void write(byte[] b, int off, int len) - throws IOException - { - super.write(b, off, len); - hash.update(b, off, len); - } - - @SuppressWarnings("NumericCastThatLosesPrecision") - @Override - public void write(int b) - throws IOException - { - oneByte[0] = (byte) b; - write(oneByte, 0, 1); - } - - @Override - public void close() - throws IOException - { - if (closed) { - return; - } - closed = true; - - try (Closer closer = Closer.create()) { - closer.register(super::close); - closer.register(() -> getFD().sync()); - closer.register(this::flush); - } - - // extremely paranoid code to detect a broken local file system - try (InputStream in = new FileInputStream(file)) { - if (hash.hash() != XxHash64.hash(in)) { - throw new IOException("File is corrupt after write"); - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/UuidUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/UuidUtil.java deleted file mode 100644 index 724d52c2c4c9..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/UuidUtil.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import org.jdbi.v3.core.argument.AbstractArgumentFactory; -import org.jdbi.v3.core.argument.Argument; -import org.jdbi.v3.core.argument.internal.strategies.LoggableBinderArgument; -import org.jdbi.v3.core.config.ConfigRegistry; -import org.jdbi.v3.core.mapper.ColumnMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.nio.ByteBuffer; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Types; -import java.util.UUID; - -public final class UuidUtil -{ - private UuidUtil() {} - - public static final class UuidArgumentFactory - extends AbstractArgumentFactory - { - public UuidArgumentFactory() - { - super(Types.VARBINARY); - } - - @Override - protected Argument build(UUID uuid, ConfigRegistry config) - { - return new LoggableBinderArgument<>(uuid, (statement, index, value) -> - statement.setBytes(index, uuidToBytes(value))); - } - } - - public static final class UuidColumnMapper - implements ColumnMapper - { - @Override - public UUID map(ResultSet r, int index, StatementContext ctx) - throws SQLException - { - return uuidFromBytes(r.getBytes(index)); - } - } - - public static UUID uuidFromBytes(byte[] bytes) - { - ByteBuffer buffer = ByteBuffer.wrap(bytes); - long msb = buffer.getLong(); - long lsb = buffer.getLong(); - return new UUID(msb, lsb); - } - - public static byte[] uuidToBytes(UUID uuid) - { - return ByteBuffer.allocate(16) - .putLong(uuid.getMostSignificantBits()) - .putLong(uuid.getLeastSignificantBits()) - .array(); - } - - /** - * @param uuidSlice textual representation of UUID - * @return byte representation of UUID - * @throws IllegalArgumentException if uuidSlice is not a valid string representation of UUID - */ - public static Slice uuidStringToBytes(Slice uuidSlice) - { - UUID uuid = UUID.fromString(uuidSlice.toStringUtf8()); - byte[] bytes = uuidToBytes(uuid); - return Slices.wrappedBuffer(bytes); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/BaseRaptorConnectorTest.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/BaseRaptorConnectorTest.java deleted file mode 100644 index 9793c30f44dd..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/BaseRaptorConnectorTest.java +++ /dev/null @@ -1,1111 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.SetMultimap; -import io.trino.Session; -import io.trino.spi.type.ArrayType; -import io.trino.testing.BaseConnectorTest; -import io.trino.testing.MaterializedResult; -import io.trino.testing.MaterializedRow; -import io.trino.testing.TestingConnectorBehavior; -import io.trino.testing.sql.TestTable; -import org.intellij.lang.annotations.Language; -import org.junit.jupiter.api.Test; - -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.StringJoiner; -import java.util.UUID; -import java.util.stream.IntStream; - -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.airlift.testing.Assertions.assertLessThan; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.testing.TestingNames.randomNameSuffix; -import static java.lang.String.format; -import static java.util.Arrays.asList; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assumptions.abort; - -public abstract class BaseRaptorConnectorTest - extends BaseConnectorTest -{ - @Override - protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) - { - return switch (connectorBehavior) { - case SUPPORTS_ADD_COLUMN_WITH_COMMENT, - SUPPORTS_COMMENT_ON_COLUMN, - SUPPORTS_COMMENT_ON_TABLE, - SUPPORTS_CREATE_MATERIALIZED_VIEW, - SUPPORTS_CREATE_SCHEMA, - SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT, - SUPPORTS_CREATE_TABLE_WITH_TABLE_COMMENT, - SUPPORTS_NOT_NULL_CONSTRAINT, - SUPPORTS_ROW_TYPE, - SUPPORTS_SET_COLUMN_TYPE, - SUPPORTS_TOPN_PUSHDOWN, - SUPPORTS_TRUNCATE -> false; - default -> super.hasBehavior(connectorBehavior); - }; - } - - @Override - protected TestTable createTableWithDefaultColumns() - { - return abort("Raptor connector does not support column default values"); - } - - @Override - protected void verifyConcurrentUpdateFailurePermissible(Exception e) - { - assertThat(e).hasMessageContaining("Table was updated by a different transaction. Please retry the operation."); - } - - @Test - @Override - public void testCharVarcharComparison() - { - assertThatThrownBy(super::testCharVarcharComparison) - .hasMessage("Unsupported type: char(3)"); - } - - @Test - @Override - public void testRenameTableAcrossSchema() - { - // Raptor allows renaming to a schema it doesn't exist https://github.com/trinodb/trino/issues/11110 - String tableName = "test_rename_old_" + randomNameSuffix(); - assertUpdate("CREATE TABLE " + tableName + " AS SELECT 123 x", 1); - - String schemaName = "test_schema_" + randomNameSuffix(); - - String renamedTable = "test_rename_new_" + randomNameSuffix(); - assertUpdate("ALTER TABLE " + tableName + " RENAME TO " + schemaName + "." + renamedTable); - - assertThat(getQueryRunner().tableExists(getSession(), tableName)).isFalse(); - assertQuery("SELECT x FROM " + schemaName + "." + renamedTable, "VALUES 123"); - - assertUpdate("DROP TABLE " + schemaName + "." + renamedTable); - - assertThat(getQueryRunner().tableExists(getSession(), tableName)).isFalse(); - assertThat(getQueryRunner().tableExists(Session.builder(getSession()).setSchema(schemaName).build(), renamedTable)).isFalse(); - } - - @Override - protected Optional filterDataMappingSmokeTestData(DataMappingTestSetup dataMappingTestSetup) - { - String typeName = dataMappingTestSetup.getTrinoTypeName(); - if (typeName.equals("tinyint") - || typeName.equals("real") - || typeName.startsWith("decimal(") - || typeName.equals("time") - || typeName.equals("time(6)") - || typeName.equals("timestamp(6)") - || typeName.equals("timestamp(3) with time zone") - || typeName.equals("timestamp(6) with time zone") - || typeName.startsWith("char(")) { - //TODO this should either work or fail cleanly - return Optional.empty(); - } - - return Optional.of(dataMappingTestSetup); - } - - @Override - protected Optional filterCaseSensitiveDataMappingTestData(DataMappingTestSetup dataMappingTestSetup) - { - String typeName = dataMappingTestSetup.getTrinoTypeName(); - if (typeName.equals("char(1)")) { - //TODO this should either work or fail cleanly - return Optional.empty(); - } - return Optional.of(dataMappingTestSetup); - } - - @Test - public void testCreateArrayTable() - { - assertUpdate("CREATE TABLE array_test AS SELECT ARRAY [1, 2, 3] AS c", 1); - assertQuery("SELECT cardinality(c) FROM array_test", "SELECT 3"); - assertUpdate("DROP TABLE array_test"); - } - - @Test - public void testMapTable() - { - assertUpdate("CREATE TABLE map_test AS SELECT MAP(ARRAY [1, 2, 3], ARRAY ['hi', 'bye', NULL]) AS c", 1); - assertQuery("SELECT c[1] FROM map_test", "SELECT 'hi'"); - assertQuery("SELECT c[3] FROM map_test", "SELECT NULL"); - assertUpdate("DROP TABLE map_test"); - } - - @Test - @Override - public void testCreateViewSchemaNotFound() - { - // TODO (https://github.com/trinodb/trino/issues/11110) Raptor connector can create new views in a schema where it doesn't exist - assertThatThrownBy(super::testCreateViewSchemaNotFound) - .hasMessageContaining("Expected query to fail: CREATE VIEW test_schema_"); - } - - @Test - public void testCreateTableViewAlreadyExists() - { - assertUpdate("CREATE VIEW view_already_exists AS SELECT 1 a"); - assertQueryFails("CREATE TABLE view_already_exists(a integer)", "View already exists: tpch.view_already_exists"); - assertQueryFails("CREATE TABLE View_Already_Exists(a integer)", "View already exists: tpch.view_already_exists"); - assertQueryFails("CREATE TABLE view_already_exists AS SELECT 1 a", "View already exists: tpch.view_already_exists"); - assertQueryFails("CREATE TABLE View_Already_Exists AS SELECT 1 a", "View already exists: tpch.view_already_exists"); - assertUpdate("DROP VIEW view_already_exists"); - } - - @Test - public void testCreateViewTableAlreadyExists() - { - assertUpdate("CREATE TABLE table_already_exists (id integer)"); - assertQueryFails("CREATE VIEW table_already_exists AS SELECT 1 a", ".*Table already exists: 'raptor.tpch.table_already_exists'"); - assertQueryFails("CREATE VIEW Table_Already_Exists AS SELECT 1 a", ".*Table already exists: 'raptor.tpch.table_already_exists'"); - assertQueryFails("CREATE OR REPLACE VIEW table_already_exists AS SELECT 1 a", ".*Table already exists: 'raptor.tpch.table_already_exists'"); - assertQueryFails("CREATE OR REPLACE VIEW Table_Already_Exists AS SELECT 1 a", ".*Table already exists: 'raptor.tpch.table_already_exists'"); - assertUpdate("DROP TABLE table_already_exists"); - } - - @Test - public void testInsertSelectDecimal() - { - assertUpdate("CREATE TABLE test_decimal(short_decimal DECIMAL(5,2), long_decimal DECIMAL(25,20))"); - assertUpdate("INSERT INTO test_decimal VALUES(DECIMAL '123.45', DECIMAL '12345.12345678901234567890')", "VALUES(1)"); - assertUpdate("INSERT INTO test_decimal VALUES(NULL, NULL)", "VALUES(1)"); - assertQuery("SELECT * FROM test_decimal", "VALUES (123.45, 12345.12345678901234567890), (NULL, NULL)"); - assertUpdate("DROP TABLE test_decimal"); - } - - @Test - public void testShardUuidHiddenColumn() - { - assertUpdate("CREATE TABLE test_shard_uuid AS SELECT orderdate, orderkey FROM orders", "SELECT count(*) FROM orders"); - - MaterializedResult actualResults = computeActual("SELECT *, \"$shard_uuid\" FROM test_shard_uuid"); - assertThat(actualResults.getTypes()).isEqualTo(ImmutableList.of(DATE, BIGINT, SHARD_UUID_COLUMN_TYPE)); - UUID arbitraryUuid = null; - for (MaterializedRow row : actualResults.getMaterializedRows()) { - Object uuid = row.getField(2); - assertInstanceOf(uuid, String.class); - arbitraryUuid = UUID.fromString((String) uuid); - } - assertThat(arbitraryUuid).isNotNull(); - - actualResults = computeActual(format("SELECT * FROM test_shard_uuid where \"$shard_uuid\" = '%s'", arbitraryUuid)); - assertThat(actualResults.getMaterializedRows().size()) - .isNotEqualTo(0); - actualResults = computeActual("SELECT * FROM test_shard_uuid where \"$shard_uuid\" = 'foo'"); - assertThat(actualResults.getMaterializedRows().size()).isEqualTo(0); - } - - @Test - public void testBucketNumberHiddenColumn() - { - assertUpdate("" + - "CREATE TABLE test_bucket_number " + - "WITH (bucket_count = 50, bucketed_on = ARRAY ['orderkey']) " + - "AS SELECT * FROM orders", - "SELECT count(*) FROM orders"); - - MaterializedResult actualResults = computeActual("SELECT DISTINCT \"$bucket_number\" FROM test_bucket_number"); - assertThat(actualResults.getTypes()).isEqualTo(ImmutableList.of(INTEGER)); - Set actual = actualResults.getMaterializedRows().stream() - .map(row -> row.getField(0)) - .collect(toSet()); - assertThat(actual).isEqualTo(IntStream.range(0, 50).boxed().collect(toSet())); - } - - @Test - public void testNoBucketNumberHiddenColumn() - { - assertThatThrownBy(() -> { - assertUpdate("CREATE TABLE test_no_bucket_number (test bigint)"); - computeActual("SELECT DISTINCT \"$bucket_number\" FROM test_no_bucket_number"); - }) - .isInstanceOf(RuntimeException.class) - .hasMessageMatching(".*Column '\\$bucket_number' cannot be resolved"); - } - - @Test - public void testShardingByTemporalDateColumn() - { - // Make sure we have at least 2 different orderdate. - assertThat(computeActual("SELECT count(DISTINCT orderdate) >= 2 FROM orders WHERE orderdate < date '1992-02-08'").getOnlyValue()).isEqualTo(true); - - assertUpdate("CREATE TABLE test_shard_temporal_date " + - "WITH (temporal_column = 'orderdate') AS " + - "SELECT orderdate, orderkey " + - "FROM orders " + - "WHERE orderdate < date '1992-02-08'", - "SELECT count(*) " + - "FROM orders " + - "WHERE orderdate < date '1992-02-08'"); - - MaterializedResult results = computeActual("SELECT orderdate, \"$shard_uuid\" FROM test_shard_temporal_date"); - - // Each shard will only contain data of one date. - SetMultimap shardDateMap = HashMultimap.create(); - for (MaterializedRow row : results.getMaterializedRows()) { - shardDateMap.put((String) row.getField(1), (LocalDate) row.getField(0)); - } - - for (Collection dates : shardDateMap.asMap().values()) { - assertThat(dates.size()).isEqualTo(1); - } - - // Make sure we have all the rows - assertQuery("SELECT orderdate, orderkey FROM test_shard_temporal_date", - "SELECT orderdate, orderkey FROM orders WHERE orderdate < date '1992-02-08'"); - } - - @Test - public void testShardingByTemporalDateColumnBucketed() - { - // Make sure we have at least 2 different orderdate. - assertThat(computeActual("SELECT count(DISTINCT orderdate) >= 2 FROM orders WHERE orderdate < date '1992-02-08'").getOnlyValue()).isEqualTo(true); - - assertUpdate("CREATE TABLE test_shard_temporal_date_bucketed " + - "WITH (temporal_column = 'orderdate', bucket_count = 10, bucketed_on = ARRAY ['orderkey']) AS " + - "SELECT orderdate, orderkey " + - "FROM orders " + - "WHERE orderdate < date '1992-02-08'", - "SELECT count(*) " + - "FROM orders " + - "WHERE orderdate < date '1992-02-08'"); - - MaterializedResult results = computeActual("SELECT orderdate, \"$shard_uuid\" FROM test_shard_temporal_date_bucketed"); - - // Each shard will only contain data of one date. - SetMultimap shardDateMap = HashMultimap.create(); - for (MaterializedRow row : results.getMaterializedRows()) { - shardDateMap.put((String) row.getField(1), (LocalDate) row.getField(0)); - } - - for (Collection dates : shardDateMap.asMap().values()) { - assertThat(dates.size()).isEqualTo(1); - } - - // Make sure we have all the rows - assertQuery("SELECT orderdate, orderkey FROM test_shard_temporal_date_bucketed", - "SELECT orderdate, orderkey FROM orders WHERE orderdate < date '1992-02-08'"); - } - - @Test - public void testShardingByTemporalTimestampColumn() - { - assertUpdate("CREATE TABLE test_shard_temporal_timestamp(col1 BIGINT, col2 TIMESTAMP) WITH (temporal_column = 'col2')"); - - int rows = 20; - StringJoiner joiner = new StringJoiner(", ", "INSERT INTO test_shard_temporal_timestamp VALUES ", ""); - for (int i = 0; i < rows; i++) { - joiner.add(format("(%s, TIMESTAMP '2016-08-08 01:00' + interval '%s' hour)", i, i * 4)); - } - - assertUpdate(joiner.toString(), format("VALUES(%s)", rows)); - - MaterializedResult results = computeActual("SELECT cast(cast(col2 as DATE) as VARCHAR), \"$shard_uuid\" FROM test_shard_temporal_timestamp"); - assertThat(results.getRowCount()).isEqualTo(rows); - - // Each shard will only contain data of one date. - SetMultimap shardDateMap = HashMultimap.create(); - for (MaterializedRow row : results.getMaterializedRows()) { - shardDateMap.put((String) row.getField(1), (String) row.getField(0)); - } - - for (Collection dates : shardDateMap.asMap().values()) { - assertThat(dates.size()).isEqualTo(1); - } - - // Ensure one shard can contain different timestamps from the same day - assertLessThan(shardDateMap.size(), rows); - } - - @Test - public void testShardingByTemporalTimestampColumnBucketed() - { - assertUpdate("" + - "CREATE TABLE test_shard_temporal_timestamp_bucketed(col1 BIGINT, col2 TIMESTAMP) " + - "WITH (temporal_column = 'col2', bucket_count = 3, bucketed_on = ARRAY ['col1'])"); - - int rows = 100; - StringJoiner joiner = new StringJoiner(", ", "INSERT INTO test_shard_temporal_timestamp_bucketed VALUES ", ""); - for (int i = 0; i < rows; i++) { - joiner.add(format("(%s, TIMESTAMP '2016-08-08 01:00' + interval '%s' hour)", i, i)); - } - - assertUpdate(joiner.toString(), format("VALUES(%s)", rows)); - - MaterializedResult results = computeActual("" + - "SELECT cast(cast(col2 as DATE) as VARCHAR), \"$shard_uuid\" " + - "FROM test_shard_temporal_timestamp_bucketed"); - - assertThat(results.getRowCount()).isEqualTo(rows); - - // Each shard will only contain data of one date. - SetMultimap shardDateMap = HashMultimap.create(); - for (MaterializedRow row : results.getMaterializedRows()) { - shardDateMap.put((String) row.getField(1), (String) row.getField(0)); - } - - for (Collection dates : shardDateMap.asMap().values()) { - assertThat(dates.size()).isEqualTo(1); - } - - // Ensure one shard can contain different timestamps from the same day - assertLessThan(shardDateMap.size(), rows); - } - - @Test - public void testTableProperties() - { - computeActual("CREATE TABLE test_table_properties_1 (foo BIGINT, bar BIGINT, ds DATE) WITH (ordering=array['foo','bar'], temporal_column='ds')"); - computeActual("CREATE TABLE test_table_properties_2 (foo BIGINT, bar BIGINT, ds DATE) WITH (ORDERING=array['foo','bar'], TEMPORAL_COLUMN='ds')"); - } - - @Test - public void testShardsSystemTable() - { - assertQuery("" + - "SELECT table_schema, table_name, sum(row_count)\n" + - "FROM system.shards\n" + - "WHERE table_schema = 'tpch'\n" + - " AND table_name IN ('orders', 'region')\n" + - "GROUP BY 1, 2", - "" + - "SELECT 'tpch', 'orders', (SELECT count(*) FROM orders)\n" + - "UNION ALL\n" + - "SELECT 'tpch', 'region', (SELECT count(*) FROM region)"); - } - - @Test - public void testShardsSystemTableWithTemporalColumn() - { - // Make sure we have rows in the selected range - assertThat(computeActual("SELECT count(*) >= 1 FROM orders WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08'").getOnlyValue()).isEqualTo(true); - - // Create a table that has DATE type temporal column - assertUpdate("CREATE TABLE test_shards_system_table_date_temporal\n" + - "WITH (temporal_column = 'orderdate') AS\n" + - "SELECT orderdate, orderkey\n" + - "FROM orders\n" + - "WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08'", - "SELECT count(*)\n" + - "FROM orders\n" + - "WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08'"); - - // Create a table that has TIMESTAMP type temporal column - assertUpdate("CREATE TABLE test_shards_system_table_timestamp_temporal\n" + - "WITH (temporal_column = 'ordertimestamp') AS\n" + - "SELECT CAST (orderdate AS TIMESTAMP) AS ordertimestamp, orderkey\n" + - "FROM test_shards_system_table_date_temporal", - "SELECT count(*)\n" + - "FROM orders\n" + - "WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08'"); - - // For table with DATE type temporal column, min/max_timestamp columns must be null while min/max_date columns must not be null - assertThat(computeActual("" + - "SELECT count(*)\n" + - "FROM system.shards\n" + - "WHERE table_schema = 'tpch'\n" + - "AND table_name = 'test_shards_system_table_date_temporal'\n" + - "AND NOT \n" + - "(min_timestamp IS NULL AND max_timestamp IS NULL\n" + - "AND min_date IS NOT NULL AND max_date IS NOT NULL)").getOnlyValue()).isEqualTo(0L); - - // For table with TIMESTAMP type temporal column, min/max_date columns must be null while min/max_timestamp columns must not be null - assertThat(computeActual("" + - "SELECT count(*)\n" + - "FROM system.shards\n" + - "WHERE table_schema = 'tpch'\n" + - "AND table_name = 'test_shards_system_table_timestamp_temporal'\n" + - "AND NOT\n" + - "(min_date IS NULL AND max_date IS NULL\n" + - "AND min_timestamp IS NOT NULL AND max_timestamp IS NOT NULL)").getOnlyValue()).isEqualTo(0L); - - // Test date predicates in table with DATE temporal column - assertQuery("" + - "SELECT table_schema, table_name, sum(row_count)\n" + - "FROM system.shards \n" + - "WHERE table_schema = 'tpch'\n" + - "AND table_name = 'test_shards_system_table_date_temporal'\n" + - "AND min_date >= date '1992-01-01'\n" + - "AND max_date <= date '1992-02-08'\n" + - "GROUP BY 1, 2", - "" + - "SELECT 'tpch', 'test_shards_system_table_date_temporal',\n" + - "(SELECT count(*) FROM orders WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08')"); - - // Test timestamp predicates in table with TIMESTAMP temporal column - assertQuery("" + - "SELECT table_schema, table_name, sum(row_count)\n" + - "FROM system.shards \n" + - "WHERE table_schema = 'tpch'\n" + - "AND table_name = 'test_shards_system_table_timestamp_temporal'\n" + - "AND min_timestamp >= timestamp '1992-01-01'\n" + - "AND max_timestamp <= timestamp '1992-02-08'\n" + - "GROUP BY 1, 2", - "" + - "SELECT 'tpch', 'test_shards_system_table_timestamp_temporal',\n" + - "(SELECT count(*) FROM orders WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08')"); - } - - @Test - public void testColumnRangesSystemTable() - { - assertQuery("SELECT orderkey_min, orderkey_max, custkey_min, custkey_max, orderdate_min, orderdate_max FROM \"orders$column_ranges\"", - "SELECT min(orderkey), max(orderkey), min(custkey), max(custkey), min(orderdate), max(orderdate) FROM orders"); - - assertQuery("SELECT orderkey_min, orderkey_max FROM \"orders$column_ranges\"", - "SELECT min(orderkey), max(orderkey) FROM orders"); - - // No such table test - assertQueryFails("SELECT * FROM \"no_table$column_ranges\"", ".*'raptor\\.tpch\\.\"no_table\\$column_ranges\"' does not exist.*"); - - // No range column for DOUBLE, INTEGER or VARCHAR - assertQueryFails("SELECT totalprice_min FROM \"orders$column_ranges\"", ".*Column 'totalprice_min' cannot be resolved.*"); - assertQueryFails("SELECT shippriority_min FROM \"orders$column_ranges\"", ".*Column 'shippriority_min' cannot be resolved.*"); - assertQueryFails("SELECT orderstatus_min FROM \"orders$column_ranges\"", ".*Column 'orderstatus_min' cannot be resolved.*"); - assertQueryFails("SELECT orderpriority_min FROM \"orders$column_ranges\"", ".*Column 'orderpriority_min' cannot be resolved.*"); - assertQueryFails("SELECT clerk_min FROM \"orders$column_ranges\"", ".*Column 'clerk_min' cannot be resolved.*"); - assertQueryFails("SELECT comment_min FROM \"orders$column_ranges\"", ".*Column 'comment_min' cannot be resolved.*"); - - // Empty table - assertUpdate("CREATE TABLE column_ranges_test (a BIGINT, b BIGINT)"); - assertQuery("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", "SELECT NULL, NULL, NULL, NULL"); - - // Table with NULL values - assertUpdate("INSERT INTO column_ranges_test VALUES (1, NULL)", 1); - assertQuery("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", "SELECT 1, 1, NULL, NULL"); - assertUpdate("INSERT INTO column_ranges_test VALUES (NULL, 99)", 1); - assertQuery("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", "SELECT 1, 1, 99, 99"); - assertUpdate("INSERT INTO column_ranges_test VALUES (50, 50)", 1); - assertQuery("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", "SELECT 1, 50, 50, 99"); - - // Drop table - assertUpdate("DROP TABLE column_ranges_test"); - assertQueryFails("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", - ".*'raptor\\.tpch\\.\"column_ranges_test\\$column_ranges\"' does not exist.*"); - } - - @Test - public void testCreateBucketedTable() - { - assertUpdate("DROP TABLE IF EXISTS orders_bucketed"); - assertUpdate("" + - "CREATE TABLE orders_bucketed " + - "WITH (bucket_count = 50, bucketed_on = ARRAY ['orderkey']) " + - "AS SELECT * FROM orders", - "SELECT count(*) FROM orders"); - - assertQuery("SELECT * FROM orders_bucketed", "SELECT * FROM orders"); - assertQuery("SELECT count(*) FROM orders_bucketed", "SELECT count(*) FROM orders"); - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed", "SELECT 50"); - assertQuery("SELECT count(DISTINCT \"$bucket_number\") FROM orders_bucketed", "SELECT 50"); - - assertUpdate("INSERT INTO orders_bucketed SELECT * FROM orders", "SELECT count(*) FROM orders"); - - assertQuery("SELECT * FROM orders_bucketed", "SELECT * FROM orders UNION ALL SELECT * FROM orders"); - assertQuery("SELECT count(*) FROM orders_bucketed", "SELECT count(*) * 2 FROM orders"); - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed", "SELECT 50 * 2"); - assertQuery("SELECT count(DISTINCT \"$bucket_number\") FROM orders_bucketed", "SELECT 50"); - - assertQuery("SELECT count(*) FROM orders_bucketed a JOIN orders_bucketed b USING (orderkey)", "SELECT count(*) * 4 FROM orders"); - - assertUpdate("DELETE FROM orders_bucketed WHERE orderkey = 37", 2); - assertQuery("SELECT count(*) FROM orders_bucketed", "SELECT (count(*) * 2) - 2 FROM orders"); - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed", "SELECT 50 * 2"); - assertQuery("SELECT count(DISTINCT \"$bucket_number\") FROM orders_bucketed", "SELECT 50"); - - assertUpdate("DROP TABLE orders_bucketed"); - } - - @Test - public void testCreateBucketedTableLike() - { - assertUpdate("" + - "CREATE TABLE orders_bucketed_original (" + - " orderkey bigint" + - ", custkey bigint" + - ") " + - "WITH (bucket_count = 50, bucketed_on = ARRAY['orderkey'])"); - - assertUpdate("" + - "CREATE TABLE orders_bucketed_like (" + - " orderdate date" + - ", LIKE orders_bucketed_original INCLUDING PROPERTIES" + - ")"); - - assertUpdate("INSERT INTO orders_bucketed_like SELECT orderdate, orderkey, custkey FROM orders", "SELECT count(*) FROM orders"); - assertUpdate("INSERT INTO orders_bucketed_like SELECT orderdate, orderkey, custkey FROM orders", "SELECT count(*) FROM orders"); - - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed_like", "SELECT 50 * 2"); - - assertUpdate("DROP TABLE orders_bucketed_original"); - assertUpdate("DROP TABLE orders_bucketed_like"); - } - - @Test - public void testBucketingMixedTypes() - { - assertUpdate("" + - "CREATE TABLE orders_bucketed_mixed " + - "WITH (bucket_count = 50, bucketed_on = ARRAY ['custkey', 'clerk', 'shippriority']) " + - "AS SELECT * FROM orders", - "SELECT count(*) FROM orders"); - - assertQuery("SELECT * FROM orders_bucketed_mixed", "SELECT * FROM orders"); - assertQuery("SELECT count(*) FROM orders_bucketed_mixed", "SELECT count(*) FROM orders"); - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed_mixed", "SELECT 50"); - assertQuery("SELECT count(DISTINCT \"$bucket_number\") FROM orders_bucketed_mixed", "SELECT 50"); - } - - @Test - @Override - public void testShowCreateTable() - { - super.testShowCreateTable(); - - String createTableSql = format("" + - "CREATE TABLE %s.%s.%s (\n" + - " c1 bigint,\n" + - " c2 double,\n" + - " \"c 3\" varchar,\n" + - " \"c'4\" array(bigint),\n" + - " c5 map(bigint, varchar),\n" + - " c6 bigint,\n" + - " c7 timestamp(3)\n" + - ")\n" + - "WITH (\n" + - " bucket_count = 32,\n" + - " bucketed_on = ARRAY['c1','c6'],\n" + - " ordering = ARRAY['c6','c1'],\n" + - " temporal_column = 'c7'\n" + - ")", - getSession().getCatalog().get(), getSession().getSchema().get(), "test_show_create_table"); - assertUpdate(createTableSql); - - MaterializedResult actualResult = computeActual("SHOW CREATE TABLE test_show_create_table"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE " + getSession().getSchema().get() + ".test_show_create_table"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE " + getSession().getCatalog().get() + "." + getSession().getSchema().get() + ".test_show_create_table"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - // With organization enabled - createTableSql = format("" + - "CREATE TABLE %s.%s.%s (\n" + - " c1 bigint,\n" + - " c2 double,\n" + - " \"c 3\" varchar,\n" + - " \"c'4\" array(bigint),\n" + - " c5 map(bigint, varchar),\n" + - " c6 bigint,\n" + - " c7 timestamp(3)\n" + - ")\n" + - "WITH (\n" + - " bucket_count = 32,\n" + - " bucketed_on = ARRAY['c1','c6'],\n" + - " ordering = ARRAY['c6','c1'],\n" + - " organized = true\n" + - ")", - getSession().getCatalog().get(), getSession().getSchema().get(), "test_show_create_table_organized"); - assertUpdate(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE test_show_create_table_organized"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE " + getSession().getSchema().get() + ".test_show_create_table_organized"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE " + getSession().getCatalog().get() + "." + getSession().getSchema().get() + ".test_show_create_table_organized"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - createTableSql = format("" + - "CREATE TABLE %s.%s.%s (\n" + - " \"c\"\"1\" bigint,\n" + - " c2 double,\n" + - " \"c 3\" varchar,\n" + - " \"c'4\" array(bigint),\n" + - " c5 map(bigint, varchar)\n" + - ")", - getSession().getCatalog().get(), getSession().getSchema().get(), "\"test_show_create_table\"\"2\""); - assertUpdate(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE \"test_show_create_table\"\"2\""); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - } - - @Test - @Override - public void testCreateTableSchemaNotFound() - { - // TODO (https://github.com/trinodb/trino/issues/11110) Raptor connector can create new tables in a schema where it doesn't exist - assertThatThrownBy(super::testCreateTableSchemaNotFound) - .hasMessageContaining("Expected query to fail: CREATE TABLE test_schema_"); - } - - @Test - @Override - public void testCreateTableAsSelectSchemaNotFound() - { - // TODO (https://github.com/trinodb/trino/issues/11110) Raptor connector can create new tables in a schema where it doesn't exist - assertThatThrownBy(super::testCreateTableAsSelectSchemaNotFound) - .hasMessageContaining("Expected query to fail: CREATE TABLE test_schema_"); - } - - @Test - public void testTablesSystemTable() - { - assertUpdate("" + - "CREATE TABLE system_tables_test0 (c00 timestamp, c01 varchar, c02 double, c03 bigint, c04 bigint)"); - assertUpdate("" + - "CREATE TABLE system_tables_test1 (c10 timestamp, c11 varchar, c12 double, c13 bigint, c14 bigint) " + - "WITH (temporal_column = 'c10')"); - assertUpdate("" + - "CREATE TABLE system_tables_test2 (c20 timestamp, c21 varchar, c22 double, c23 bigint, c24 bigint) " + - "WITH (temporal_column = 'c20', ordering = ARRAY['c22', 'c21'])"); - assertUpdate("" + - "CREATE TABLE system_tables_test3 (c30 timestamp, c31 varchar, c32 double, c33 bigint, c34 bigint) " + - "WITH (temporal_column = 'c30', bucket_count = 40, bucketed_on = ARRAY ['c34', 'c33'])"); - assertUpdate("" + - "CREATE TABLE system_tables_test4 (c40 timestamp, c41 varchar, c42 double, c43 bigint, c44 bigint) " + - "WITH (temporal_column = 'c40', ordering = ARRAY['c41', 'c42'], distribution_name = 'test_distribution', bucket_count = 50, bucketed_on = ARRAY ['c43', 'c44'])"); - assertUpdate("" + - "CREATE TABLE system_tables_test5 (c50 timestamp, c51 varchar, c52 double, c53 bigint, c54 bigint) " + - "WITH (ordering = ARRAY['c51', 'c52'], distribution_name = 'test_distribution', bucket_count = 50, bucketed_on = ARRAY ['c53', 'c54'], organized = true)"); - - MaterializedResult actualResults = computeActual("SELECT * FROM system.tables"); - assertThat(actualResults.getTypes()).isEqualTo(ImmutableList.builder() - .add(VARCHAR) // table_schema - .add(VARCHAR) // table_name - .add(VARCHAR) // temporal_column - .add(new ArrayType(VARCHAR)) // ordering_columns - .add(VARCHAR) // distribution_name - .add(BIGINT) // bucket_count - .add(new ArrayType(VARCHAR)) // bucket_columns - .add(BOOLEAN) // organized - .build()); - Map map = actualResults.getMaterializedRows().stream() - .filter(row -> ((String) row.getField(1)).startsWith("system_tables_test")) - .collect(toImmutableMap(row -> ((String) row.getField(1)), identity())); - assertThat(map.size()).isEqualTo(6); - assertThat(map.get("system_tables_test0").getFields()).isEqualTo(asList("tpch", "system_tables_test0", null, null, null, null, null, Boolean.FALSE)); - assertThat(map.get("system_tables_test1").getFields()).isEqualTo(asList("tpch", "system_tables_test1", "c10", null, null, null, null, Boolean.FALSE)); - assertThat(map.get("system_tables_test2").getFields()).isEqualTo(asList("tpch", "system_tables_test2", "c20", ImmutableList.of("c22", "c21"), null, null, null, Boolean.FALSE)); - assertThat(map.get("system_tables_test3").getFields()).isEqualTo(asList("tpch", "system_tables_test3", "c30", null, null, 40L, ImmutableList.of("c34", "c33"), Boolean.FALSE)); - assertThat(map.get("system_tables_test4").getFields()).isEqualTo(asList("tpch", "system_tables_test4", "c40", ImmutableList.of("c41", "c42"), "test_distribution", 50L, ImmutableList.of("c43", "c44"), Boolean.FALSE)); - assertThat(map.get("system_tables_test5").getFields()).isEqualTo(asList("tpch", "system_tables_test5", null, ImmutableList.of("c51", "c52"), "test_distribution", 50L, ImmutableList.of("c53", "c54"), Boolean.TRUE)); - - actualResults = computeActual("SELECT * FROM system.tables WHERE table_schema = 'tpch'"); - long actualRowCount = actualResults.getMaterializedRows().stream() - .filter(row -> ((String) row.getField(1)).startsWith("system_tables_test")) - .count(); - assertThat(actualRowCount).isEqualTo(6); - - actualResults = computeActual("SELECT * FROM system.tables WHERE table_name = 'system_tables_test3'"); - assertThat(actualResults.getMaterializedRows().size()).isEqualTo(1); - - actualResults = computeActual("SELECT * FROM system.tables WHERE table_schema = 'tpch' and table_name = 'system_tables_test3'"); - assertThat(actualResults.getMaterializedRows().size()).isEqualTo(1); - - actualResults = computeActual("" + - "SELECT distribution_name, bucket_count, bucketing_columns, ordering_columns, temporal_column, organized " + - "FROM system.tables " + - "WHERE table_schema = 'tpch' and table_name = 'system_tables_test3'"); - assertThat(actualResults.getTypes()).isEqualTo(ImmutableList.of(VARCHAR, BIGINT, new ArrayType(VARCHAR), new ArrayType(VARCHAR), VARCHAR, BOOLEAN)); - assertThat(actualResults.getMaterializedRows().size()).isEqualTo(1); - - assertUpdate("DROP TABLE system_tables_test0"); - assertUpdate("DROP TABLE system_tables_test1"); - assertUpdate("DROP TABLE system_tables_test2"); - assertUpdate("DROP TABLE system_tables_test3"); - assertUpdate("DROP TABLE system_tables_test4"); - assertUpdate("DROP TABLE system_tables_test5"); - - assertThat(computeActual("SELECT * FROM system.tables WHERE table_schema IN ('foo', 'bar')").getRowCount()).isEqualTo(0); - } - - @Test - public void testTableStatsSystemTable() - { - // basic sanity tests - assertQuery("" + - "SELECT table_schema, table_name, sum(row_count)\n" + - "FROM system.table_stats\n" + - "WHERE table_schema = 'tpch'\n" + - " AND table_name IN ('orders', 'region')\n" + - "GROUP BY 1, 2", - "" + - "SELECT 'tpch', 'orders', (SELECT count(*) FROM orders)\n" + - "UNION ALL\n" + - "SELECT 'tpch', 'region', (SELECT count(*) FROM region)"); - - assertQuery("" + - "SELECT\n" + - " bool_and(row_count >= shard_count)\n" + - ", bool_and(update_time >= create_time)\n" + - ", bool_and(table_version >= 1)\n" + - "FROM system.table_stats\n" + - "WHERE row_count > 0", - "SELECT true, true, true"); - - // create empty table - assertUpdate("CREATE TABLE test_table_stats (x bigint)"); - - @Language("SQL") String sql = "" + - "SELECT create_time, update_time, table_version," + - " shard_count, row_count, uncompressed_size\n" + - "FROM system.table_stats\n" + - "WHERE table_schema = 'tpch'\n" + - " AND table_name = 'test_table_stats'"; - MaterializedRow row = getOnlyElement(computeActual(sql).getMaterializedRows()); - - LocalDateTime createTime = (LocalDateTime) row.getField(0); - LocalDateTime updateTime1 = (LocalDateTime) row.getField(1); - assertThat(createTime).isEqualTo(updateTime1); - - assertThat(row.getField(2)).isEqualTo(1L); // table_version - assertThat(row.getField(3)).isEqualTo(0L); // shard_count - assertThat(row.getField(4)).isEqualTo(0L); // row_count - long size1 = (long) row.getField(5); // uncompressed_size - - // insert - assertUpdate("INSERT INTO test_table_stats VALUES (1), (2), (3), (4)", 4); - row = getOnlyElement(computeActual(sql).getMaterializedRows()); - - assertThat(row.getField(0)).isEqualTo(createTime); - LocalDateTime updateTime2 = (LocalDateTime) row.getField(1); - assertLessThan(updateTime1, updateTime2); - - assertThat(row.getField(2)).isEqualTo(2L); // table_version - assertGreaterThanOrEqual((Long) row.getField(3), 1L); // shard_count - assertThat(row.getField(4)).isEqualTo(4L); // row_count - long size2 = (long) row.getField(5); // uncompressed_size - assertGreaterThan(size2, size1); - - // delete - assertUpdate("DELETE FROM test_table_stats WHERE x IN (2, 4)", 2); - row = getOnlyElement(computeActual(sql).getMaterializedRows()); - - assertThat(row.getField(0)).isEqualTo(createTime); - LocalDateTime updateTime3 = (LocalDateTime) row.getField(1); - assertLessThan(updateTime2, updateTime3); - - assertThat(row.getField(2)).isEqualTo(3L); // table_version - assertGreaterThanOrEqual((Long) row.getField(3), 1L); // shard_count - assertThat(row.getField(4)).isEqualTo(2L); // row_count - long size3 = (long) row.getField(5); // uncompressed_Size - assertLessThan(size3, size2); - - // add column - assertUpdate("ALTER TABLE test_table_stats ADD COLUMN y bigint"); - row = getOnlyElement(computeActual(sql).getMaterializedRows()); - - assertThat(row.getField(0)).isEqualTo(createTime); - assertLessThan(updateTime3, (LocalDateTime) row.getField(1)); - - assertThat(row.getField(2)).isEqualTo(4L); // table_version - assertThat(row.getField(4)).isEqualTo(2L); // row_count - assertThat(row.getField(5)).isEqualTo(size3); // uncompressed_size - - // cleanup - assertUpdate("DROP TABLE test_table_stats"); - } - - @Test - public void testAlterTable() - { - assertUpdate("CREATE TABLE test_alter_table (c1 bigint, c2 bigint)"); - assertUpdate("INSERT INTO test_alter_table VALUES (1, 1), (1, 2), (1, 3), (1, 4)", 4); - assertUpdate("INSERT INTO test_alter_table VALUES (11, 1), (11, 2)", 2); - - assertUpdate("ALTER TABLE test_alter_table ADD COLUMN c3 bigint"); - assertQueryFails("ALTER TABLE test_alter_table DROP COLUMN c3", "Cannot drop the column which has the largest column ID in the table"); - assertUpdate("INSERT INTO test_alter_table VALUES (2, 1, 1), (2, 2, 2), (2, 3, 3), (2, 4, 4)", 4); - assertUpdate("INSERT INTO test_alter_table VALUES (22, 1, 1), (22, 2, 2), (22, 4, 4)", 3); - - // Do a partial delete on a shard that does not contain newly added column - assertUpdate("DELETE FROM test_alter_table WHERE c1 = 1 and c2 = 1", 1); - // Then drop a full shard that does not contain newly added column - assertUpdate("DELETE FROM test_alter_table WHERE c1 = 11", 2); - - // Drop a column from middle of table - assertUpdate("ALTER TABLE test_alter_table DROP COLUMN c2"); - assertUpdate("INSERT INTO test_alter_table VALUES (3, 1), (3, 2), (3, 3), (3, 4)", 4); - - // Do a partial delete on a shard that contains column already dropped - assertUpdate("DELETE FROM test_alter_table WHERE c1 = 2 and c3 = 1", 1); - // Then drop a full shard that contains column already dropped - assertUpdate("DELETE FROM test_alter_table WHERE c1 = 22", 3); - - assertUpdate("DROP TABLE test_alter_table"); - } - - @Override - protected void verifyConcurrentAddColumnFailurePermissible(Exception e) - { - assertThat(e) - .hasMessageContaining("Failed to perform metadata operation") - .cause() - .hasMessageMatching( - "(?s).*SQLIntegrityConstraintViolationException.*" + - "|.*Unique index or primary key violation.*" + - "|.*Deadlock found when trying to get lock; try restarting transaction.*"); - } - - @Override - protected OptionalInt maxTableNameLength() - { - return OptionalInt.of(255); - } - - @Override - protected void verifyTableNameLengthFailurePermissible(Throwable e) - { - assertThat(e).hasMessage("Failed to perform metadata operation"); - } - - @Override - protected OptionalInt maxColumnNameLength() - { - return OptionalInt.of(255); - } - - @Override - protected void verifyColumnNameLengthFailurePermissible(Throwable e) - { - assertThat(e).hasMessage("Failed to perform metadata operation"); - } - - @Test - public void testMergeMultipleOperationsUnbucketed() - { - String targetTable = "merge_multiple_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (customer VARCHAR, purchases INT, zipcode INT, spouse VARCHAR, address VARCHAR)", targetTable)); - testMergeMultipleOperationsInternal(targetTable, 32); - } - - @Test - public void testMergeMultipleOperationsBucketed() - { - String targetTable = "merge_multiple_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (customer VARCHAR, purchases INT, zipcode INT, spouse VARCHAR, address VARCHAR)" + - " WITH (bucket_count=4, bucketed_on=ARRAY['customer'])", targetTable)); - testMergeMultipleOperationsInternal(targetTable, 32); - } - - private void testMergeMultipleOperationsInternal(String targetTable, int targetCustomerCount) - { - String originalInsertFirstHalf = IntStream.range(1, targetCustomerCount / 2) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jan_%s', '%s Poe Ct')", intValue, 1000, 91000, intValue, intValue)) - .collect(joining(", ")); - String originalInsertSecondHalf = IntStream.range(targetCustomerCount / 2, targetCustomerCount) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jan_%s', '%s Poe Ct')", intValue, 2000, 92000, intValue, intValue)) - .collect(joining(", ")); - - assertUpdate(format("INSERT INTO %s (customer, purchases, zipcode, spouse, address) VALUES %s, %s", targetTable, originalInsertFirstHalf, originalInsertSecondHalf), targetCustomerCount - 1); - - String firstMergeSource = IntStream.range(targetCustomerCount / 2, targetCustomerCount) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jill_%s', '%s Eop Ct')", intValue, 3000, 83000, intValue, intValue)) - .collect(joining(", ")); - - @Language("SQL") String sql = format("MERGE INTO %s t USING (SELECT * FROM (VALUES %s)) AS s(customer, purchases, zipcode, spouse, address)", targetTable, firstMergeSource) + - " ON t.customer = s.customer" + - " WHEN MATCHED THEN UPDATE SET purchases = s.purchases, zipcode = s.zipcode, spouse = s.spouse, address = s.address"; - assertUpdate(sql, targetCustomerCount / 2); - - assertQuery( - "SELECT customer, purchases, zipcode, spouse, address FROM " + targetTable, - format("SELECT * FROM (VALUES %s, %s) AS v(customer, purchases, zipcode, spouse, address)", originalInsertFirstHalf, firstMergeSource)); - - String nextInsert = IntStream.range(targetCustomerCount, targetCustomerCount * 3 / 2) - .mapToObj(intValue -> format("('jack_%s', %s, %s, 'jan_%s', '%s Poe Ct')", intValue, 4000, 74000, intValue, intValue)) - .collect(joining(", ")); - assertUpdate(format("INSERT INTO %s (customer, purchases, zipcode, spouse, address) VALUES %s", targetTable, nextInsert), targetCustomerCount / 2); - - String secondMergeSource = IntStream.range(1, targetCustomerCount * 3 / 2) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jen_%s', '%s Poe Ct')", intValue, 5000, 85000, intValue, intValue)) - .collect(joining(", ")); - - assertUpdate(format("MERGE INTO %s t USING (SELECT * FROM (VALUES %s)) AS s(customer, purchases, zipcode, spouse, address)", targetTable, secondMergeSource) + - " ON t.customer = s.customer" + - " WHEN MATCHED AND t.zipcode = 91000 THEN DELETE" + - " WHEN MATCHED AND s.zipcode = 85000 THEN UPDATE SET zipcode = 60000" + - " WHEN MATCHED THEN UPDATE SET zipcode = s.zipcode, spouse = s.spouse, address = s.address" + - " WHEN NOT MATCHED THEN INSERT (customer, purchases, zipcode, spouse, address) VALUES(s.customer, s.purchases, s.zipcode, s.spouse, s.address)", - targetCustomerCount * 3 / 2 - 1); - - String updatedBeginning = IntStream.range(targetCustomerCount / 2, targetCustomerCount) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jill_%s', '%s Eop Ct')", intValue, 3000, 60000, intValue, intValue)) - .collect(joining(", ")); - String updatedMiddle = IntStream.range(targetCustomerCount, targetCustomerCount * 3 / 2) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jen_%s', '%s Poe Ct')", intValue, 5000, 85000, intValue, intValue)) - .collect(joining(", ")); - String updatedEnd = IntStream.range(targetCustomerCount, targetCustomerCount * 3 / 2) - .mapToObj(intValue -> format("('jack_%s', %s, %s, 'jan_%s', '%s Poe Ct')", intValue, 4000, 74000, intValue, intValue)) - .collect(joining(", ")); - - assertQuery( - "SELECT customer, purchases, zipcode, spouse, address FROM " + targetTable, - format("SELECT * FROM (VALUES %s, %s, %s) AS v(customer, purchases, zipcode, spouse, address)", updatedBeginning, updatedMiddle, updatedEnd)); - assertUpdate("DROP TABLE " + targetTable); - } - - @Test - public void testMergeSimpleQueryBucketed() - { - String targetTable = "merge_simple_target_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count=7, bucketed_on=ARRAY['address'])", targetTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 5, 'Antioch'), ('Bill', 7, 'Buena'), ('Carol', 3, 'Cambridge'), ('Dave', 11, 'Devon')", targetTable), 4); - - @Language("SQL") String query = format("MERGE INTO %s t USING ", targetTable) + - "(SELECT * FROM (VALUES ('Aaron', 6, 'Arches'), ('Carol', 9, 'Centreville'), ('Dave', 11, 'Darbyshire'), ('Ed', 7, 'Etherville'))) AS s(customer, purchases, address)" + - " " + - "ON (t.customer = s.customer)" + - " WHEN MATCHED AND s.address = 'Centreville' THEN DELETE" + - " WHEN MATCHED THEN UPDATE SET purchases = s.purchases + t.purchases, address = s.address" + - " WHEN NOT MATCHED THEN INSERT (customer, purchases, address) VALUES(s.customer, s.purchases, s.address)"; - assertUpdate(query, 4); - - assertQuery("SELECT * FROM " + targetTable, "VALUES ('Aaron', 11, 'Arches'), ('Bill', 7, 'Buena'), ('Dave', 22, 'Darbyshire'), ('Ed', 7, 'Etherville')"); - } - - @Test - public void testMergeMultipleRows() - { - testMergeMultipleRowsMatchFails("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR)"); - testMergeMultipleRowsMatchFails("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['customer'])"); - testMergeMultipleRowsMatchFails("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 4, bucketed_on = ARRAY['address'])"); - testMergeMultipleRowsMatchFails("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 4, bucketed_on = ARRAY['address', 'purchases', 'customer'])"); - } - - private void testMergeMultipleRowsMatchFails(String createTableSql) - { - String targetTable = "merge_all_matches_deleted_target_" + randomNameSuffix(); - assertUpdate(format(createTableSql, targetTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 5, 'Antioch'), ('Bill', 7, 'Antioch')", targetTable), 2); - - String sourceTable = "merge_all_matches_deleted_source_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR)", sourceTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 6, 'Adelphi'), ('Aaron', 8, 'Ashland')", sourceTable), 2); - - assertThatThrownBy(() -> computeActual(format("MERGE INTO %s t USING %s s ON (t.customer = s.customer)", targetTable, sourceTable) + - " WHEN MATCHED THEN UPDATE SET address = s.address")) - .hasMessage("One MERGE target table row matched more than one source row"); - - assertUpdate(format("MERGE INTO %s t USING %s s ON (t.customer = s.customer)", targetTable, sourceTable) + - " WHEN MATCHED AND s.address = 'Adelphi' THEN UPDATE SET address = s.address", - 1); - assertQuery("SELECT customer, purchases, address FROM " + targetTable, "VALUES ('Aaron', 5, 'Adelphi'), ('Bill', 7, 'Antioch')"); - - assertUpdate("DROP TABLE " + sourceTable); - assertUpdate("DROP TABLE " + targetTable); - } - - @Test - public void testMergeWithDifferentBucketing() - { - testMergeWithDifferentBucketing( - "target_and_source_with_different_bucketing_counts", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 5, bucketed_on = ARRAY['customer'])", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['purchases', 'address'])"); - testMergeWithDifferentBucketing( - "target_and_source_with_different_bucketing_columns", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['address'])", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['customer'])"); - testMergeWithDifferentBucketing( - "target_flat_source_bucketed_by_customer", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR)", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['customer'])"); - testMergeWithDifferentBucketing( - "target_bucketed_by_customer_source_flat", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['customer'])", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR)"); - } - - private void testMergeWithDifferentBucketing(String testDescription, String createTargetTableSql, String createSourceTableSql) - { - String targetTable = format("%s_target_%s", testDescription, randomNameSuffix()); - assertUpdate(format(createTargetTableSql, targetTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 5, 'Antioch'), ('Bill', 7, 'Buena'), ('Carol', 3, 'Cambridge'), ('Dave', 11, 'Devon')", targetTable), 4); - - String sourceTable = format("%s_source_%s", testDescription, randomNameSuffix()); - assertUpdate(format(createSourceTableSql, sourceTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 6, 'Arches'), ('Ed', 7, 'Etherville'), ('Carol', 9, 'Centreville'), ('Dave', 11, 'Darbyshire')", sourceTable), 4); - - @Language("SQL") String sql = format("MERGE INTO %s t USING %s s ON (t.customer = s.customer)", targetTable, sourceTable) + - " WHEN MATCHED AND s.address = 'Centreville' THEN DELETE" + - " WHEN MATCHED THEN UPDATE SET purchases = s.purchases + t.purchases, address = s.address" + - " WHEN NOT MATCHED THEN INSERT (customer, purchases, address) VALUES(s.customer, s.purchases, s.address)"; - - assertUpdate(sql, 4); - - assertQuery("SELECT customer, purchases, address FROM " + targetTable, "VALUES ('Aaron', 11, 'Arches'), ('Ed', 7, 'Etherville'), ('Bill', 7, 'Buena'), ('Dave', 22, 'Darbyshire')"); - - assertUpdate("DROP TABLE " + sourceTable); - assertUpdate("DROP TABLE " + targetTable); - } - - @Test - public void testMergeOverManySplits() - { - String targetTable = "merge_delete_select_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (orderkey bigint, custkey bigint, orderstatus varchar(1), totalprice double, orderdate date, orderpriority varchar(15), clerk varchar(15), shippriority integer, comment varchar(79))", targetTable)); - - assertUpdate(format("INSERT INTO %s SELECT * FROM tpch.\"sf0.1\".orders", targetTable), 150000); - - @Language("SQL") String sql = format("MERGE INTO %s t USING (SELECT * FROM tpch.\"sf0.1\".orders) s ON (t.orderkey = s.orderkey)", targetTable) + - " WHEN MATCHED AND mod(s.orderkey, 3) = 0 THEN UPDATE SET totalprice = t.totalprice + s.totalprice" + - " WHEN MATCHED AND mod(s.orderkey, 3) = 1 THEN DELETE"; - - assertUpdate(sql, 100_000); - - assertQuery(format("SELECT count(*) FROM %s t WHERE mod(t.orderkey, 3) = 1", targetTable), "VALUES (0)"); - - assertUpdate("DROP TABLE " + targetTable); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/DatabaseTesting.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/DatabaseTesting.java deleted file mode 100644 index 1ce2b73256de..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/DatabaseTesting.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.sqlobject.SqlObjectPlugin; - -import java.util.concurrent.ThreadLocalRandom; - -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; - -public final class DatabaseTesting -{ - private DatabaseTesting() {} - - public static Jdbi createTestingJdbi() - { - return Jdbi.create("jdbc:h2:mem:test" + System.nanoTime() + ThreadLocalRandom.current().nextLong()) - .installPlugin(new SqlObjectPlugin()) - .registerRowMapper(new TableColumn.Mapper(TESTING_TYPE_MANAGER)) - .registerRowMapper(new Distribution.Mapper(TESTING_TYPE_MANAGER)); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/RaptorQueryRunner.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/RaptorQueryRunner.java deleted file mode 100644 index d0e567f220d3..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/RaptorQueryRunner.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.errorprone.annotations.CanIgnoreReturnValue; -import io.airlift.log.Logger; -import io.trino.Session; -import io.trino.SystemSessionProperties; -import io.trino.connector.CatalogServiceProvider; -import io.trino.metadata.QualifiedObjectName; -import io.trino.metadata.SessionPropertyManager; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.plugin.tpch.TpchPlugin; -import io.trino.spi.session.PropertyMetadata; -import io.trino.testing.DistributedQueryRunner; -import io.trino.testing.QueryRunner; -import io.trino.tpch.TpchTable; -import org.intellij.lang.annotations.Language; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import static io.airlift.testing.Closeables.closeAllSuppress; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.trino.testing.QueryAssertions.copyTpchTables; -import static io.trino.testing.TestingHandles.createTestCatalogHandle; -import static io.trino.testing.TestingSession.testSessionBuilder; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public final class RaptorQueryRunner -{ - private RaptorQueryRunner() {} - - private static final Logger log = Logger.get(RaptorQueryRunner.class); - - public static Builder builder() - { - return new Builder() - .addConnectorProperty("metadata.db.type", "h2") - .addConnectorProperty("metadata.db.filename", createTempDirectory("raptor-db").toString()) - .addConnectorProperty("storage.data-directory", createTempDirectory("raptor-data").toString()) - .addConnectorProperty("storage.max-shard-rows", "2000") - .addConnectorProperty("backup.provider", "file") - .addConnectorProperty("backup.directory", createTempDirectory("raptor-backup").toString()); - } - - public static final class Builder - extends DistributedQueryRunner.Builder - { - private final Map connectorProperties = new HashMap<>(); - private boolean bucketed; - private List> initialTables = ImmutableList.of(); - - private Builder() - { - super(createSession()); - } - - @CanIgnoreReturnValue - public Builder addConnectorProperty(String key, String value) - { - this.connectorProperties.put(key, value); - return this; - } - - @CanIgnoreReturnValue - public Builder enableBucketed() - { - this.bucketed = true; - return this; - } - - @CanIgnoreReturnValue - public Builder setInitialTables(Iterable> initialTables) - { - this.initialTables = ImmutableList.copyOf(requireNonNull(initialTables, "initialTables is null")); - return this; - } - - @Override - public DistributedQueryRunner build() - throws Exception - { - DistributedQueryRunner queryRunner = super.build(); - try { - queryRunner.installPlugin(new TpchPlugin()); - queryRunner.createCatalog("tpch", "tpch"); - - queryRunner.installPlugin(new RaptorPlugin()); - queryRunner.createCatalog("raptor", "raptor_legacy", connectorProperties); - - copyTables(queryRunner, "tpch", createSession(), bucketed, initialTables); - - return queryRunner; - } - catch (Throwable e) { - closeAllSuppress(e, queryRunner); - throw e; - } - } - } - - public static void copyTables(QueryRunner queryRunner, String catalog, Session session, boolean bucketed, List> tables) - { - String schema = TINY_SCHEMA_NAME; - if (!bucketed) { - copyTpchTables(queryRunner, catalog, schema, session, tables); - return; - } - - ImmutableMap.Builder, String> tablesMapBuilder = ImmutableMap.builder(); - for (TpchTable table : tables) { - if (table.equals(TpchTable.ORDERS)) { - tablesMapBuilder.put(TpchTable.ORDERS, "bucket_count = 25, bucketed_on = ARRAY['orderkey'], distribution_name = 'order'"); - } - else if (table.equals(TpchTable.LINE_ITEM)) { - tablesMapBuilder.put(TpchTable.LINE_ITEM, "bucket_count = 25, bucketed_on = ARRAY['orderkey'], distribution_name = 'order'"); - } - else if (table.equals(TpchTable.PART)) { - tablesMapBuilder.put(TpchTable.PART, "bucket_count = 20, bucketed_on = ARRAY['partkey'], distribution_name = 'part'"); - } - else if (table.equals(TpchTable.PART_SUPPLIER)) { - tablesMapBuilder.put(TpchTable.PART_SUPPLIER, "bucket_count = 20, bucketed_on = ARRAY['partkey'], distribution_name = 'part'"); - } - else if (table.equals(TpchTable.SUPPLIER)) { - tablesMapBuilder.put(TpchTable.SUPPLIER, "bucket_count = 10, bucketed_on = ARRAY['suppkey']"); - } - else if (table.equals(TpchTable.CUSTOMER)) { - tablesMapBuilder.put(TpchTable.CUSTOMER, "bucket_count = 10, bucketed_on = ARRAY['custkey']"); - } - else if (table.equals(TpchTable.NATION)) { - tablesMapBuilder.put(TpchTable.NATION, ""); - } - else if (table.equals(TpchTable.REGION)) { - tablesMapBuilder.put(TpchTable.REGION, ""); - } - else { - throw new IllegalArgumentException("Unsupported table: " + table); - } - } - Map, String> tablesMap = tablesMapBuilder.buildOrThrow(); - - log.info("Loading data from %s.%s...", catalog, schema); - long startTime = System.nanoTime(); - for (Entry, String> entry : tablesMap.entrySet()) { - copyTable(queryRunner, catalog, session, schema, entry.getKey(), entry.getValue()); - } - log.info("Loading from %s.%s complete in %s", catalog, schema, nanosSince(startTime)); - } - - private static void copyTable(QueryRunner queryRunner, String catalog, Session session, String schema, TpchTable table, String properties) - { - QualifiedObjectName source = new QualifiedObjectName(catalog, schema, table.getTableName()); - String target = table.getTableName(); - - String with = properties.isEmpty() ? "" : format(" WITH (%s)", properties); - @Language("SQL") String sql = format("CREATE TABLE %s%s AS SELECT * FROM %s", target, with, source); - - log.info("Running import for %s", target); - long start = System.nanoTime(); - long rows = queryRunner.execute(session, sql).getUpdateCount().getAsLong(); - log.info("Imported %s rows for %s in %s", rows, target, nanosSince(start)); - } - - public static Session createSession() - { - return createSession("tpch"); - } - - public static Session createSession(String schema) - { - SessionPropertyManager sessionPropertyManager = new SessionPropertyManager( - ImmutableSet.of(new SystemSessionProperties()), - CatalogServiceProvider.singleton( - createTestCatalogHandle("raptor"), - Maps.uniqueIndex(new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties(), PropertyMetadata::getName))); - return testSessionBuilder(sessionPropertyManager) - .setCatalog("raptor") - .setSchema(schema) - .setSystemProperty("enable_intermediate_aggregations", "true") - .build(); - } - - public static Path createTempDirectory(String name) - { - try { - Path tempDirectory = Files.createTempDirectory(name); - tempDirectory.toFile().deleteOnExit(); - return tempDirectory.toAbsolutePath(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static void main(String[] args) - throws Exception - { - QueryRunner queryRunner = RaptorQueryRunner.builder() - .addCoordinatorProperty("http-server.http.port", "8080") - .build(); - Logger log = Logger.get(RaptorQueryRunner.class); - log.info("======== SERVER STARTED ========"); - log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketFunction.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketFunction.java deleted file mode 100644 index 1e389c2662fe..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketFunction.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import static io.trino.block.BlockAssertions.createIntsBlock; -import static io.trino.block.BlockAssertions.createLongsBlock; -import static io.trino.block.BlockAssertions.createStringsBlock; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestRaptorBucketFunction -{ - @Test - public void testBigint() - { - BucketFunction function = bucketFunction(50, BIGINT); - assertThat(getBucket(function, createLongsBlock(123456789012L))).isEqualTo(12); - assertThat(getBucket(function, createLongsBlock(454345325))).isEqualTo(16); - assertThat(getBucket(function, createLongsBlock(365363))).isEqualTo(42); - assertThat(getBucket(function, createLongsBlock(45645747))).isEqualTo(41); - assertThat(getBucket(function, createLongsBlock(3244))).isEqualTo(29); - - function = bucketFunction(2, BIGINT); - assertThat(getBucket(function, createLongsBlock(123456789012L))).isEqualTo(0); - assertThat(getBucket(function, createLongsBlock(454345325))).isEqualTo(0); - assertThat(getBucket(function, createLongsBlock(365363))).isEqualTo(0); - assertThat(getBucket(function, createLongsBlock(45645747))).isEqualTo(1); - assertThat(getBucket(function, createLongsBlock(3244))).isEqualTo(1); - } - - @Test - public void testInteger() - { - BucketFunction function = bucketFunction(50, INTEGER); - assertThat(getBucket(function, createIntsBlock(454345325))).isEqualTo(16); - assertThat(getBucket(function, createIntsBlock(365363))).isEqualTo(42); - assertThat(getBucket(function, createIntsBlock(45645747))).isEqualTo(41); - assertThat(getBucket(function, createIntsBlock(3244))).isEqualTo(29); - } - - @Test - public void testVarchar() - { - BucketFunction function = bucketFunction(50, createUnboundedVarcharType()); - assertThat(getBucket(function, createStringsBlock("lorem ipsum"))).isEqualTo(2); - assertThat(getBucket(function, createStringsBlock("lorem"))).isEqualTo(26); - assertThat(getBucket(function, createStringsBlock("ipsum"))).isEqualTo(3); - assertThat(getBucket(function, createStringsBlock("hello"))).isEqualTo(19); - } - - @Test - public void testVarcharBigint() - { - BucketFunction function = bucketFunction(50, createUnboundedVarcharType(), BIGINT); - assertThat(getBucket(function, createStringsBlock("lorem ipsum"), createLongsBlock(123456789012L))).isEqualTo(24); - assertThat(getBucket(function, createStringsBlock("lorem"), createLongsBlock(454345325))).isEqualTo(32); - assertThat(getBucket(function, createStringsBlock("ipsum"), createLongsBlock(365363))).isEqualTo(21); - assertThat(getBucket(function, createStringsBlock("hello"), createLongsBlock(45645747))).isEqualTo(34); - assertThat(getBucket(function, createStringsBlock("world"), createLongsBlock(3244))).isEqualTo(4); - } - - private static int getBucket(BucketFunction function, Block... blocks) - { - return function.getBucket(new Page(blocks), 0); - } - - private static BucketFunction bucketFunction(int bucketCount, Type... types) - { - return new RaptorBucketFunction(bucketCount, ImmutableList.copyOf(types)); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketedConnectorTest.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketedConnectorTest.java deleted file mode 100644 index c1a1ceaff756..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketedConnectorTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.testing.QueryRunner; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class TestRaptorBucketedConnectorTest - extends BaseRaptorConnectorTest -{ - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - return RaptorQueryRunner.builder() - .enableBucketed() - .addConnectorProperty("storage.compaction-enabled", "false") - .setInitialTables(REQUIRED_TPCH_TABLES) - .build(); - } - - @Test - @Override - public void testShowCreateTable() - { - assertThat(computeActual("SHOW CREATE TABLE orders").getOnlyValue()) - .isEqualTo("CREATE TABLE raptor.tpch.orders (\n" + - " orderkey bigint,\n" + - " custkey bigint,\n" + - " orderstatus varchar(1),\n" + - " totalprice double,\n" + - " orderdate date,\n" + - " orderpriority varchar(15),\n" + - " clerk varchar(15),\n" + - " shippriority integer,\n" + - " comment varchar(79)\n" + - ")\n" + - "WITH (\n" + - " bucket_count = 25,\n" + - " bucketed_on = ARRAY['orderkey'],\n" + - " distribution_name = 'order'\n" + - ")"); - } - - @Test - public void testShardsSystemTableBucketNumber() - { - assertQuery("" + - "SELECT count(DISTINCT bucket_number)\n" + - "FROM system.shards\n" + - "WHERE table_schema = 'tpch'\n" + - " AND table_name = 'orders'", - "SELECT 25"); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java deleted file mode 100644 index aeb493179200..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.slice.Slice; -import io.trino.operator.PagesIndex; -import io.trino.operator.PagesIndexPageSorter; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.NodeManager; -import io.trino.spi.Page; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.SqlDate; -import io.trino.spi.type.SqlTimestamp; -import io.trino.spi.type.Type; -import io.trino.testing.MaterializedResult; -import io.trino.testing.TestingConnectorSession; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.nio.file.Files; -import java.util.Collection; -import java.util.Optional; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.operator.scalar.timestamp.VarcharToTimestampCast.castToShortTimestamp; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.TestRaptorStorageManager.createRaptorStorageManager; -import static io.trino.spi.connector.RetryMode.NO_RETRIES; -import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.testing.TestingPageSinkId.TESTING_PAGE_SINK_ID; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static io.trino.util.DateTimeUtils.parseDate; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestRaptorConnector -{ - private static final ConnectorSession SESSION = TestingConnectorSession.builder() - .setPropertyMetadata(new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties()) - .build(); - - private Handle dummyHandle; - private MetadataDao metadataDao; - private File dataDir; - private RaptorConnector connector; - - @BeforeEach - public void setup() - throws Exception - { - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - metadataDao = dbi.onDemand(MetadataDao.class); - createTablesWithRetry(dbi); - dataDir = Files.createTempDirectory(null).toFile(); - - CatalogName connectorId = new CatalogName("test"); - NodeManager nodeManager = new TestingNodeManager(); - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - ShardManager shardManager = createShardManager(dbi); - StorageManager storageManager = createRaptorStorageManager(dbi, dataDir); - StorageManagerConfig config = new StorageManagerConfig(); - connector = new RaptorConnector( - new LifeCycleManager(ImmutableList.of(), null), - new TestingNodeManager(), - new RaptorMetadataFactory(dbi, shardManager), - new RaptorSplitManager(connectorId, nodeSupplier, shardManager, false), - new RaptorPageSourceProvider(storageManager), - new RaptorPageSinkProvider(storageManager, - new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)), - config), - new RaptorNodePartitioningProvider(nodeSupplier), - new RaptorSessionProperties(config), - new RaptorTableProperties(TESTING_TYPE_MANAGER), - ImmutableSet.of(), - Optional.empty(), - dbi); - } - - @AfterEach - public void tearDown() - throws Exception - { - dummyHandle.close(); - dummyHandle = null; - deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); - } - - @Test - public void testMaintenanceBlocked() - { - long tableId1 = createTable("test1"); - long tableId2 = createTable("test2"); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isFalse(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isFalse(); - - // begin delete for table1 - ConnectorTransactionHandle txn1 = beginTransaction(); - ConnectorTableHandle handle1 = getTableHandle(connector.getMetadata(SESSION, txn1), "test1"); - connector.getMetadata(SESSION, txn1).beginMerge(SESSION, handle1, NO_RETRIES); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isTrue(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isFalse(); - - // begin delete for table2 - ConnectorTransactionHandle txn2 = beginTransaction(); - ConnectorTableHandle handle2 = getTableHandle(connector.getMetadata(SESSION, txn2), "test2"); - connector.getMetadata(SESSION, txn2).beginMerge(SESSION, handle2, NO_RETRIES); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isTrue(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isTrue(); - - // begin another delete for table1 - ConnectorTransactionHandle txn3 = beginTransaction(); - ConnectorTableHandle handle3 = getTableHandle(connector.getMetadata(SESSION, txn3), "test1"); - connector.getMetadata(SESSION, txn3).beginMerge(SESSION, handle3, NO_RETRIES); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isTrue(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isTrue(); - - // commit first delete for table1 - connector.commit(txn1); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isTrue(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isTrue(); - - // rollback second delete for table1 - connector.rollback(txn3); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isFalse(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isTrue(); - - // commit delete for table2 - connector.commit(txn2); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isFalse(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isFalse(); - } - - @Test - public void testMaintenanceUnblockedOnStart() - { - long tableId = createTable("test"); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId)).isFalse(); - metadataDao.blockMaintenance(tableId); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId)).isTrue(); - - connector.start(); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId)).isFalse(); - } - - @Test - public void testTemporalShardSplit() - throws Exception - { - // Same date should be in same split - assertSplitShard(DATE, "2001-08-22", "2001-08-22", 1); - - // Same date should be in different splits - assertSplitShard(DATE, "2001-08-22", "2001-08-23", 2); - - // Same timestamp should be in same split - assertSplitShard(TIMESTAMP_MILLIS, "2001-08-22 00:00:01.000", "2001-08-22 23:59:01.000", 1); - - // Same timestamp should be in different splits - assertSplitShard(TIMESTAMP_MILLIS, "2001-08-22 23:59:01.000", "2001-08-23 00:00:01.000", 2); - } - - private void assertSplitShard(Type temporalType, String min, String max, int expectedSplits) - throws Exception - { - ConnectorSession session = TestingConnectorSession.builder() - .setPropertyMetadata(new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties()) - .build(); - - ConnectorTransactionHandle transaction = beginTransaction(); - connector.getMetadata(SESSION, transaction).createTable( - SESSION, - new ConnectorTableMetadata( - new SchemaTableName("test", "test"), - ImmutableList.of(new ColumnMetadata("id", BIGINT), new ColumnMetadata("time", temporalType)), - ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "time")), - SaveMode.FAIL); - connector.commit(transaction); - - ConnectorTransactionHandle txn1 = beginTransaction(); - ConnectorTableHandle handle1 = getTableHandle(connector.getMetadata(SESSION, txn1), "test"); - ConnectorInsertTableHandle insertTableHandle = connector.getMetadata(SESSION, txn1).beginInsert(session, handle1, ImmutableList.of(), NO_RETRIES); - ConnectorPageSink raptorPageSink = connector.getPageSinkProvider().createPageSink(txn1, session, insertTableHandle, TESTING_PAGE_SINK_ID); - - Object timestamp1 = null; - Object timestamp2 = null; - if (temporalType.equals(TIMESTAMP_MILLIS)) { - timestamp1 = SqlTimestamp.newInstance(3, castToShortTimestamp(TIMESTAMP_MILLIS.getPrecision(), min), 0); - timestamp2 = SqlTimestamp.newInstance(3, castToShortTimestamp(TIMESTAMP_MILLIS.getPrecision(), max), 0); - } - else if (temporalType.equals(DATE)) { - timestamp1 = new SqlDate(parseDate(min)); - timestamp2 = new SqlDate(parseDate(max)); - } - - Page inputPage = MaterializedResult.resultBuilder(session, ImmutableList.of(BIGINT, temporalType)) - .row(1L, timestamp1) - .row(2L, timestamp2) - .build() - .toPage(); - - raptorPageSink.appendPage(inputPage); - - Collection shards = raptorPageSink.finish().get(); - assertThat(shards.size()).isEqualTo(expectedSplits); - connector.getMetadata(session, txn1).dropTable(session, handle1); - connector.commit(txn1); - } - - private long createTable(String name) - { - ConnectorTransactionHandle transaction = beginTransaction(); - connector.getMetadata(SESSION, transaction).createTable( - SESSION, - new ConnectorTableMetadata( - new SchemaTableName("test", name), - ImmutableList.of(new ColumnMetadata("id", BIGINT))), - SaveMode.FAIL); - connector.commit(transaction); - - transaction = beginTransaction(); - ConnectorTableHandle tableHandle = getTableHandle(connector.getMetadata(SESSION, transaction), name); - connector.commit(transaction); - return ((RaptorTableHandle) tableHandle).getTableId(); - } - - private ConnectorTransactionHandle beginTransaction() - { - return connector.beginTransaction(READ_COMMITTED, false, true); - } - - private static ConnectorTableHandle getTableHandle(ConnectorMetadata metadata, String name) - { - return metadata.getTableHandle(SESSION, new SchemaTableName("test", name), Optional.empty(), Optional.empty()); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnectorTest.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnectorTest.java deleted file mode 100644 index acdb1bd522c3..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnectorTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.testing.QueryRunner; - -public class TestRaptorConnectorTest - extends BaseRaptorConnectorTest -{ - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - return RaptorQueryRunner.builder() - .addConnectorProperty("storage.compaction-enabled", "false") - .setInitialTables(REQUIRED_TPCH_TABLES) - .build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorMySqlConnectorTest.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorMySqlConnectorTest.java deleted file mode 100644 index 1c322d1515aa..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorMySqlConnectorTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableMap; -import io.trino.plugin.tpch.TpchPlugin; -import io.trino.testing.DistributedQueryRunner; -import io.trino.testing.QueryRunner; -import org.junit.jupiter.api.AfterAll; -import org.testcontainers.containers.MySQLContainer; - -import java.util.Map; - -import static io.trino.plugin.raptor.legacy.RaptorQueryRunner.copyTables; -import static io.trino.plugin.raptor.legacy.RaptorQueryRunner.createSession; -import static io.trino.plugin.raptor.legacy.RaptorQueryRunner.createTempDirectory; -import static java.lang.String.format; - -public class TestRaptorMySqlConnectorTest - extends BaseRaptorConnectorTest -{ - private MySQLContainer mysqlContainer; - - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - mysqlContainer = new MySQLContainer<>("mysql:8.0.36"); - mysqlContainer.start(); - return createRaptorMySqlQueryRunner(getJdbcUrl(mysqlContainer)); - } - - @AfterAll - public final void destroy() - { - mysqlContainer.close(); - mysqlContainer = null; - } - - private static String getJdbcUrl(MySQLContainer container) - { - return format("%s?user=%s&password=%s&useSSL=false&allowPublicKeyRetrieval=true", - container.getJdbcUrl(), - container.getUsername(), - container.getPassword()); - } - - private static QueryRunner createRaptorMySqlQueryRunner(String mysqlUrl) - throws Exception - { - QueryRunner queryRunner = DistributedQueryRunner.builder(createSession("tpch")).build(); - - queryRunner.installPlugin(new TpchPlugin()); - queryRunner.createCatalog("tpch", "tpch"); - - queryRunner.installPlugin(new RaptorPlugin()); - Map raptorProperties = ImmutableMap.builder() - .put("metadata.db.type", "mysql") - .put("metadata.db.url", mysqlUrl) - .put("storage.compaction-enabled", "false") - .put("storage.data-directory", createTempDirectory("raptor-db").toString()) - .put("storage.max-shard-rows", "2000") - .put("backup.provider", "file") - .put("backup.directory", createTempDirectory("raptor-db").toString()) - .buildOrThrow(); - - queryRunner.createCatalog("raptor", "raptor_legacy", raptorProperties); - - copyTables(queryRunner, "tpch", createSession(), false, REQUIRED_TPCH_TABLES); - - return queryRunner; - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorPlugin.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorPlugin.java deleted file mode 100644 index b713617f8627..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorPlugin.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableMap; -import io.trino.spi.Plugin; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.testing.TestingConnectorContext; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.nio.file.Files; -import java.util.Map; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.testing.Assertions.assertInstanceOf; - -public class TestRaptorPlugin -{ - @Test - public void testPlugin() - throws Exception - { - Plugin plugin = new RaptorPlugin(); - ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); - assertInstanceOf(factory, RaptorConnectorFactory.class); - - File tmpDir = Files.createTempDirectory(null).toFile(); - try { - Map config = ImmutableMap.builder() - .put("metadata.db.type", "h2") - .put("metadata.db.filename", tmpDir.getAbsolutePath()) - .put("storage.data-directory", tmpDir.getAbsolutePath()) - .put("bootstrap.quiet", "true") - .buildOrThrow(); - - factory.create("test", config, new TestingConnectorContext()).shutdown(); - } - finally { - deleteRecursively(tmpDir.toPath(), ALLOW_INSECURE); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/AbstractTestBackupStore.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/AbstractTestBackupStore.java deleted file mode 100644 index 948d4ee61ea7..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/AbstractTestBackupStore.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.nio.file.Path; -import java.util.UUID; - -import static java.nio.file.Files.readAllBytes; -import static java.nio.file.Files.writeString; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThat; - -public abstract class AbstractTestBackupStore -{ - protected Path temporary; - protected T store; - - @Test - public void testBackupStore() - throws Exception - { - // backup first file - File file1 = temporary.resolve("file1").toFile(); - writeString(file1.toPath(), "hello world"); - UUID uuid1 = randomUUID(); - - assertThat(store.shardExists(uuid1)).isFalse(); - store.backupShard(uuid1, file1); - assertThat(store.shardExists(uuid1)).isTrue(); - - // backup second file - File file2 = temporary.resolve("file2").toFile(); - writeString(file2.toPath(), "bye bye"); - UUID uuid2 = randomUUID(); - - assertThat(store.shardExists(uuid2)).isFalse(); - store.backupShard(uuid2, file2); - assertThat(store.shardExists(uuid2)).isTrue(); - - // verify first file - File restore1 = temporary.resolve("restore1").toFile(); - store.restoreShard(uuid1, restore1); - assertThat(readAllBytes(file1.toPath())).isEqualTo(readAllBytes(restore1.toPath())); - - // verify second file - File restore2 = temporary.resolve("restore2").toFile(); - store.restoreShard(uuid2, restore2); - assertThat(readAllBytes(file2.toPath())).isEqualTo(readAllBytes(restore2.toPath())); - - // verify random UUID does not exist - assertThat(store.shardExists(randomUUID())).isFalse(); - - // delete first file - assertThat(store.shardExists(uuid1)).isTrue(); - assertThat(store.shardExists(uuid2)).isTrue(); - - store.deleteShard(uuid1); - store.deleteShard(uuid1); - - assertThat(store.shardExists(uuid1)).isFalse(); - assertThat(store.shardExists(uuid2)).isTrue(); - - // delete random UUID - store.deleteShard(randomUUID()); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupConfig.java deleted file mode 100644 index db9ce39cdabf..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupConfig.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class TestBackupConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(BackupConfig.class) - .setProvider(null) - .setTimeoutThreads(1000) - .setTimeout(new Duration(1, MINUTES)) - .setBackupThreads(5)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("backup.provider", "file") - .put("backup.timeout", "42s") - .put("backup.timeout-threads", "13") - .put("backup.threads", "3") - .buildOrThrow(); - - BackupConfig expected = new BackupConfig() - .setProvider("file") - .setTimeout(new Duration(42, SECONDS)) - .setTimeoutThreads(13) - .setBackupThreads(3); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupManager.java deleted file mode 100644 index a93596ab649f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupManager.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.trino.plugin.raptor.legacy.storage.BackupStats; -import io.trino.plugin.raptor.legacy.storage.FileStorageService; -import io.trino.spi.TrinoException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ThreadLocalRandom; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_ERROR; -import static java.nio.file.Files.createTempDirectory; -import static java.nio.file.Files.writeString; -import static java.util.Objects.requireNonNull; -import static java.util.UUID.randomUUID; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestBackupManager -{ - private static final UUID FAILURE_UUID = randomUUID(); - private static final UUID CORRUPTION_UUID = randomUUID(); - - private Path temporary; - private BackupStore backupStore; - private FileStorageService storageService; - private BackupManager backupManager; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - - FileBackupStore fileStore = new FileBackupStore(temporary.resolve("backup").toFile()); - fileStore.start(); - backupStore = new TestingBackupStore(fileStore); - - storageService = new FileStorageService(temporary.resolve("data").toFile()); - storageService.start(); - - backupManager = new BackupManager(Optional.of(backupStore), storageService, 5); - } - - @AfterEach - public void tearDown() - throws Exception - { - deleteRecursively(temporary, ALLOW_INSECURE); - backupManager.shutdown(); - } - - @Test - public void testSimple() - throws Exception - { - assertEmptyStagingDirectory(); - assertBackupStats(0, 0, 0); - - List> futures = new ArrayList<>(); - List uuids = new ArrayList<>(5); - for (int i = 0; i < 5; i++) { - File file = temporary.resolve("file" + i).toFile(); - writeString(file.toPath(), "hello world"); - uuids.add(randomUUID()); - - futures.add(backupManager.submit(uuids.get(i), file)); - } - futures.forEach(CompletableFuture::join); - for (UUID uuid : uuids) { - assertThat(backupStore.shardExists(uuid)).isTrue(); - } - - assertBackupStats(5, 0, 0); - assertEmptyStagingDirectory(); - } - - @Test - public void testFailure() - throws Exception - { - assertEmptyStagingDirectory(); - assertBackupStats(0, 0, 0); - - File file = temporary.resolve("failure").toFile(); - writeString(file.toPath(), "hello world"); - - assertThatThrownBy(() -> backupManager.submit(FAILURE_UUID, file).get(10, SECONDS)) - .isInstanceOfSatisfying(ExecutionException.class, wrapper -> { - TrinoException e = (TrinoException) wrapper.getCause(); - assertThat(e.getErrorCode()).isEqualTo(RAPTOR_BACKUP_ERROR.toErrorCode()); - assertThat(e.getMessage()).isEqualTo("Backup failed for testing"); - }); - - assertBackupStats(0, 1, 0); - assertEmptyStagingDirectory(); - } - - @Test - public void testCorruption() - throws Exception - { - assertEmptyStagingDirectory(); - assertBackupStats(0, 0, 0); - - File file = temporary.resolve("corrupt").toFile(); - writeString(file.toPath(), "hello world"); - - assertThatThrownBy(() -> backupManager.submit(CORRUPTION_UUID, file).get(10, SECONDS)) - .isInstanceOfSatisfying(ExecutionException.class, wrapper -> { - TrinoException e = (TrinoException) wrapper.getCause(); - assertThat(e.getErrorCode()).isEqualTo(RAPTOR_BACKUP_CORRUPTION.toErrorCode()); - assertThat(e.getMessage()).isEqualTo("Backup is corrupt after write: " + CORRUPTION_UUID); - }); - - File quarantineBase = storageService.getQuarantineFile(CORRUPTION_UUID); - assertThat(new File(quarantineBase.getPath() + ".original")).isFile(); - assertThat(new File(quarantineBase.getPath() + ".restored")).isFile(); - - assertBackupStats(0, 1, 1); - assertEmptyStagingDirectory(); - } - - private void assertEmptyStagingDirectory() - { - File staging = storageService.getStagingFile(randomUUID()).getParentFile(); - assertThat(staging.list()).isEqualTo(new String[] {}); - } - - private void assertBackupStats(int successCount, int failureCount, int corruptionCount) - { - BackupStats stats = backupManager.getStats(); - assertThat(stats.getBackupSuccess().getTotalCount()).isEqualTo(successCount); - assertThat(stats.getBackupFailure().getTotalCount()).isEqualTo(failureCount); - assertThat(stats.getBackupCorruption().getTotalCount()).isEqualTo(corruptionCount); - } - - private static class TestingBackupStore - implements BackupStore - { - private final BackupStore delegate; - - private TestingBackupStore(BackupStore delegate) - { - this.delegate = requireNonNull(delegate, "delegate is null"); - } - - @Override - public void backupShard(UUID uuid, File source) - { - if (uuid.equals(FAILURE_UUID)) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Backup failed for testing"); - } - delegate.backupShard(uuid, source); - } - - @Override - public void restoreShard(UUID uuid, File target) - { - delegate.restoreShard(uuid, target); - if (uuid.equals(CORRUPTION_UUID)) { - corruptFile(target); - } - } - - @Override - public boolean deleteShard(UUID uuid) - { - return delegate.deleteShard(uuid); - } - - @Override - public boolean shardExists(UUID uuid) - { - return delegate.shardExists(uuid); - } - - private static void corruptFile(File path) - { - // flip a bit at a random offset - try (RandomAccessFile file = new RandomAccessFile(path, "rw")) { - if (file.length() == 0) { - throw new RuntimeException("file is empty"); - } - long offset = ThreadLocalRandom.current().nextLong(file.length()); - file.seek(offset); - int value = file.read() ^ 0x01; - file.seek(offset); - file.write(value); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupConfig.java deleted file mode 100644 index a08e89f8529f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestFileBackupConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(FileBackupConfig.class) - .setBackupDirectory(null)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("backup.directory", "/backup"); - - FileBackupConfig expected = new FileBackupConfig() - .setBackupDirectory(new File("/backup")); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupStore.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupStore.java deleted file mode 100644 index 6b0060f75160..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupStore.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; - -import java.io.File; -import java.io.IOException; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; - -@TestInstance(PER_CLASS) -public class TestFileBackupStore - extends AbstractTestBackupStore -{ - @BeforeAll - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - store = new FileBackupStore(temporary.resolve("backup").toFile()); - store.start(); - } - - @AfterAll - public void tearDown() - throws Exception - { - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testFilePaths() - { - UUID uuid = UUID.fromString("701e1a79-74f7-4f56-b438-b41e8e7d019d"); - File expected = temporary.resolve("backup").resolve("70").resolve("1e").resolve(format("%s.orc", uuid)).toFile(); - assertThat(store.getBackupFile(uuid)).isEqualTo(expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupConfig.java deleted file mode 100644 index db650c9b807f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.net.URI; -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestHttpBackupConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(HttpBackupConfig.class) - .setUri(null)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("backup.http.uri", "http://example.net:8080"); - - HttpBackupConfig expected = new HttpBackupConfig() - .setUri(URI.create("http://example.net:8080")); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupStore.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupStore.java deleted file mode 100644 index 975b86c3bf6d..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupStore.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import com.google.inject.Binder; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.http.server.HttpServerInfo; -import io.airlift.http.server.testing.TestingHttpServerModule; -import io.airlift.jaxrs.JaxrsModule; -import io.airlift.json.JsonModule; -import io.airlift.node.testing.TestingNodeModule; -import io.trino.spi.NodeManager; -import io.trino.testing.TestingNodeManager; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; - -import java.io.IOException; -import java.net.URI; -import java.util.Map; -import java.util.function.Supplier; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static com.google.inject.util.Modules.override; -import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; -import static java.nio.file.Files.createTempDirectory; - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -@Execution(ExecutionMode.SAME_THREAD) -public class TestHttpBackupStore - extends AbstractTestBackupStore -{ - private LifeCycleManager lifeCycleManager; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - - Map properties = ImmutableMap.of("backup.http.uri", "http://localhost:8080"); - - Bootstrap app = new Bootstrap( - new TestingNodeModule(), - new TestingHttpServerModule(), - new JsonModule(), - new JaxrsModule(), - binder -> jaxrsBinder(binder).bind(TestingHttpBackupResource.class), - binder -> binder.bind(NodeManager.class).toInstance(new TestingNodeManager()), - override(new HttpBackupModule()).with(new TestingModule())); - - Injector injector = app - .setRequiredConfigurationProperties(properties) - .doNotInitializeLogging() - .quiet() - .initialize(); - - lifeCycleManager = injector.getInstance(LifeCycleManager.class); - - store = injector.getInstance(BackupStore.class); - } - - @AfterEach - public void teardown() - throws IOException - { - deleteRecursively(temporary, ALLOW_INSECURE); - if (lifeCycleManager != null) { - lifeCycleManager.stop(); - } - } - - private static class TestingModule - implements Module - { - @Override - public void configure(Binder binder) {} - - @Provides - @Singleton - @ForHttpBackup - public Supplier createBackupUriSupplier(HttpServerInfo serverInfo) - { - return serverInfo::getHttpUri; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestingHttpBackupResource.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestingHttpBackupResource.java deleted file mode 100644 index 0d6a237f7f8b..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestingHttpBackupResource.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.airlift.slice.Slices; -import io.airlift.slice.XxHash64; -import io.trino.server.GoneException; -import io.trino.spi.NodeManager; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.ForbiddenException; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.HEAD; -import jakarta.ws.rs.HeaderParam; -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.PUT; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.Context; -import jakarta.ws.rs.core.Response; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import static io.trino.plugin.raptor.legacy.backup.HttpBackupStore.CONTENT_XXH64; -import static io.trino.plugin.raptor.legacy.backup.HttpBackupStore.TRINO_ENVIRONMENT; -import static jakarta.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM; -import static java.lang.Long.parseUnsignedLong; -import static java.util.Objects.requireNonNull; - -@Path("/") -public class TestingHttpBackupResource -{ - private final String environment; - - @GuardedBy("this") - private final Map shards = new HashMap<>(); - - @Inject - public TestingHttpBackupResource(NodeManager nodeManager) - { - this(nodeManager.getEnvironment()); - } - - public TestingHttpBackupResource(String environment) - { - this.environment = requireNonNull(environment, "environment is null"); - } - - @HEAD - @Path("{uuid}") - public synchronized Response headRequest( - @HeaderParam(TRINO_ENVIRONMENT) String environment, - @PathParam("uuid") UUID uuid) - { - checkEnvironment(environment); - if (!shards.containsKey(uuid)) { - throw new NotFoundException(); - } - if (shards.get(uuid) == null) { - throw new GoneException(); - } - return Response.noContent().build(); - } - - @GET - @Path("{uuid}") - @Produces(APPLICATION_OCTET_STREAM) - public synchronized Response getRequest( - @HeaderParam(TRINO_ENVIRONMENT) String environment, - @PathParam("uuid") UUID uuid) - { - checkEnvironment(environment); - if (!shards.containsKey(uuid)) { - throw new NotFoundException(); - } - byte[] bytes = shards.get(uuid); - if (bytes == null) { - throw new GoneException(); - } - return Response.ok(bytes).build(); - } - - @PUT - @Path("{uuid}") - public synchronized Response putRequest( - @HeaderParam(TRINO_ENVIRONMENT) String environment, - @HeaderParam(CONTENT_XXH64) String hexHash, - @Context HttpServletRequest request, - @PathParam("uuid") UUID uuid, - byte[] bytes) - { - checkEnvironment(environment); - if ((request.getContentLength() < 0) || (bytes.length != request.getContentLength())) { - throw new BadRequestException(); - } - if (parseUnsignedLong(hexHash, 16) != XxHash64.hash(Slices.wrappedBuffer(bytes))) { - throw new BadRequestException(); - } - if (shards.containsKey(uuid)) { - byte[] existing = shards.get(uuid); - if ((existing == null) || !Arrays.equals(bytes, existing)) { - throw new ForbiddenException(); - } - } - shards.put(uuid, bytes); - return Response.noContent().build(); - } - - @DELETE - @Path("{uuid}") - public synchronized Response deleteRequest( - @HeaderParam(TRINO_ENVIRONMENT) String environment, - @PathParam("uuid") UUID uuid) - { - checkEnvironment(environment); - if (!shards.containsKey(uuid)) { - throw new NotFoundException(); - } - if (shards.get(uuid) == null) { - throw new GoneException(); - } - shards.put(uuid, null); - return Response.noContent().build(); - } - - private void checkEnvironment(String environment) - { - if (!this.environment.equals(environment)) { - throw new ForbiddenException(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/ShardNode.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/ShardNode.java deleted file mode 100644 index 3087dd72fdc2..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/ShardNode.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import java.util.Objects; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardNode -{ - private final UUID shardUuid; - private final String nodeIdentifier; - - public ShardNode(UUID shardUuid, String nodeIdentifier) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - ShardNode other = (ShardNode) obj; - return Objects.equals(this.shardUuid, other.shardUuid) && - Objects.equals(this.nodeIdentifier, other.nodeIdentifier); - } - - @Override - public int hashCode() - { - return Objects.hash(shardUuid, nodeIdentifier); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuid", shardUuid) - .add("nodeIdentifier", nodeIdentifier) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestAssignmentLimiter.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestAssignmentLimiter.java deleted file mode 100644 index f319ddf6184a..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestAssignmentLimiter.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableSet; -import io.airlift.testing.TestingTicker; -import io.airlift.units.Duration; -import io.trino.spi.ErrorCodeSupplier; -import org.junit.jupiter.api.Test; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_REASSIGNMENT_DELAY; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_REASSIGNMENT_THROTTLE; -import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; -import static java.util.concurrent.TimeUnit.MINUTES; - -public class TestAssignmentLimiter -{ - @Test - public void testLimiter() - { - TestingTicker ticker = new TestingTicker(); - AssignmentLimiter limiter = new AssignmentLimiter( - ImmutableSet::of, - ticker, - new Duration(5, MINUTES), - new Duration(10, MINUTES)); - - // 4:00 - assertCheckFails(limiter, "A", RAPTOR_REASSIGNMENT_DELAY); - - // 4:01 - ticker.increment(1, MINUTES); - assertCheckFails(limiter, "A", RAPTOR_REASSIGNMENT_DELAY); - assertCheckFails(limiter, "B", RAPTOR_REASSIGNMENT_DELAY); - - // 4:05 - ticker.increment(4, MINUTES); - limiter.checkAssignFrom("A"); - assertCheckFails(limiter, "B", RAPTOR_REASSIGNMENT_DELAY); - - // 4:06 - ticker.increment(1, MINUTES); - assertCheckFails(limiter, "B", RAPTOR_REASSIGNMENT_THROTTLE); - assertCheckFails(limiter, "C", RAPTOR_REASSIGNMENT_DELAY); - - // 4:14 - ticker.increment(8, MINUTES); - assertCheckFails(limiter, "B", RAPTOR_REASSIGNMENT_THROTTLE); - assertCheckFails(limiter, "C", RAPTOR_REASSIGNMENT_THROTTLE); - - // 4:15 - ticker.increment(1, MINUTES); - limiter.checkAssignFrom("B"); - assertCheckFails(limiter, "C", RAPTOR_REASSIGNMENT_THROTTLE); - - // 4:24 - ticker.increment(9, MINUTES); - assertCheckFails(limiter, "C", RAPTOR_REASSIGNMENT_THROTTLE); - - // 4:25 - ticker.increment(1, MINUTES); - limiter.checkAssignFrom("A"); - - // 4:30 - ticker.increment(5, MINUTES); - limiter.checkAssignFrom("A"); - limiter.checkAssignFrom("B"); - limiter.checkAssignFrom("C"); - } - - private static void assertCheckFails(AssignmentLimiter limiter, String node, ErrorCodeSupplier expected) - { - assertTrinoExceptionThrownBy(() -> limiter.checkAssignFrom(node)) - .hasErrorCode(expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseConfig.java deleted file mode 100644 index b9f8daa3e5ff..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestDatabaseConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(DatabaseConfig.class) - .setDatabaseType(null)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("metadata.db.type", "h2"); - - DatabaseConfig expected = new DatabaseConfig() - .setDatabaseType("h2"); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseShardManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseShardManager.java deleted file mode 100644 index 3e4c7341886a..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseShardManager.java +++ /dev/null @@ -1,818 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.base.Ticker; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; -import io.airlift.slice.Slice; -import io.airlift.testing.TestingTicker; -import io.airlift.units.Duration; -import io.trino.client.NodeVersion; -import io.trino.metadata.InternalNode; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.Node; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.predicate.ValueSet; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.result.ResultIterator; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.LocalDate; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.Ticker.systemTicker; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.collect.Iterators.concat; -import static com.google.common.collect.Iterators.transform; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.storage.ShardStats.MAX_BINARY_INDEX_SIZE; -import static io.trino.spi.StandardErrorCode.SERVER_STARTING_UP; -import static io.trino.spi.StandardErrorCode.TRANSACTION_CONFLICT; -import static io.trino.spi.predicate.Range.greaterThan; -import static io.trino.spi.predicate.Range.greaterThanOrEqual; -import static io.trino.spi.predicate.Range.lessThan; -import static io.trino.spi.predicate.Range.range; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; -import static java.lang.String.format; -import static java.time.ZoneOffset.UTC; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestDatabaseShardManager -{ - private Jdbi dbi; - private Handle dummyHandle; - private File dataDir; - private ShardManager shardManager; - - @BeforeEach - public void setup() - throws Exception - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - dataDir = Files.createTempDirectory(null).toFile(); - shardManager = createShardManager(dbi); - } - - @AfterEach - public void teardown() - throws IOException - { - dummyHandle.close(); - dummyHandle = null; - deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); - } - - @Test - public void testCommit() - { - long tableId = createTable("test"); - - List shards = ImmutableList.builder() - .add(shardInfo(UUID.randomUUID(), "node1")) - .add(shardInfo(UUID.randomUUID(), "node1")) - .add(shardInfo(UUID.randomUUID(), "node2")) - .build(); - - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - Set actual = getShardNodes(tableId, TupleDomain.all()); - assertThat(actual).isEqualTo(toShardNodes(shards)); - } - - @Test - public void testRollback() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - List shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.rollbackTransaction(transactionId); - - assertTrinoExceptionThrownBy(() -> shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0)) - .hasErrorCode(TRANSACTION_CONFLICT) - .hasMessage("Transaction commit failed. Please retry the operation."); - } - - @Test - public void testAssignShard() - { - long tableId = createTable("test"); - UUID shard = UUID.randomUUID(); - List shardNodes = ImmutableList.of(shardInfo(shard, "node1")); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); - - ShardNodes actual = getOnlyElement(getShardNodes(tableId, TupleDomain.all())); - assertThat(actual).isEqualTo(new ShardNodes(shard, ImmutableSet.of("node1"))); - - assertTrinoExceptionThrownBy(() -> shardManager.replaceShardAssignment(tableId, shard, "node2", true)) - .hasErrorCode(SERVER_STARTING_UP) - .hasMessage("Cannot reassign shards while server is starting"); - - // replace shard assignment to another node - shardManager.replaceShardAssignment(tableId, shard, "node2", false); - - actual = getOnlyElement(getShardNodes(tableId, TupleDomain.all())); - assertThat(actual).isEqualTo(new ShardNodes(shard, ImmutableSet.of("node2"))); - - // replacing shard assignment should be idempotent - shardManager.replaceShardAssignment(tableId, shard, "node2", false); - - actual = getOnlyElement(getShardNodes(tableId, TupleDomain.all())); - assertThat(actual).isEqualTo(new ShardNodes(shard, ImmutableSet.of("node2"))); - } - - @Test - public void testGetNodeBytes() - { - long tableId = createTable("test"); - OptionalInt bucketNumber = OptionalInt.empty(); - - UUID shard1 = UUID.randomUUID(); - UUID shard2 = UUID.randomUUID(); - List shardNodes = ImmutableList.of( - new ShardInfo(shard1, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 3, 33, 333, 0), - new ShardInfo(shard2, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 5, 55, 555, 0)); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); - - assertThat(getShardNodes(tableId, TupleDomain.all())).isEqualTo(ImmutableSet.of( - new ShardNodes(shard1, ImmutableSet.of("node1")), - new ShardNodes(shard2, ImmutableSet.of("node1")))); - - assertThat(shardManager.getNodeBytes()).isEqualTo(ImmutableMap.of("node1", 88L)); - - shardManager.replaceShardAssignment(tableId, shard1, "node2", false); - - assertThat(getShardNodes(tableId, TupleDomain.all())).isEqualTo(ImmutableSet.of( - new ShardNodes(shard1, ImmutableSet.of("node2")), - new ShardNodes(shard2, ImmutableSet.of("node1")))); - - assertThat(shardManager.getNodeBytes()).isEqualTo(ImmutableMap.of("node1", 55L, "node2", 33L)); - } - - @Test - public void testGetNodeTableShards() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - List nodes = ImmutableList.of("node1", "node2", "node3"); - - ImmutableList.Builder inputShards = ImmutableList.builder(); - Multimap nodeShardMap = HashMultimap.create(); - for (String node : nodes) { - UUID uuid = UUID.randomUUID(); - nodeShardMap.put(node, uuid); - inputShards.add(shardInfo(uuid, node)); - } - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, inputShards.build(), Optional.empty(), 0); - - for (String node : nodes) { - Set shardMetadata = shardManager.getNodeShards(node); - Set expectedUuids = ImmutableSet.copyOf(nodeShardMap.get(node)); - Set actualUuids = shardMetadata.stream().map(ShardMetadata::getShardUuid).collect(toSet()); - assertThat(actualUuids).isEqualTo(expectedUuids); - } - } - - @Test - public void testGetExistingShards() - { - long tableId = createTable("test"); - UUID shard1 = UUID.randomUUID(); - UUID shard2 = UUID.randomUUID(); - List shardNodes = ImmutableList.of(shardInfo(shard1, "node1"), shardInfo(shard2, "node1")); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); - Set actual = shardManager.getExistingShardUuids(tableId, ImmutableSet.of(shard1, shard2, UUID.randomUUID())); - Set expected = ImmutableSet.of(shard1, shard2); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testReplaceShardUuids() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - List nodes = ImmutableList.of("node1", "node2", "node3"); - List originalUuids = ImmutableList.of(UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID()); - - List oldShards = ImmutableList.builder() - .add(shardInfo(originalUuids.get(0), nodes.get(0))) - .add(shardInfo(originalUuids.get(1), nodes.get(1))) - .add(shardInfo(originalUuids.get(2), nodes.get(2))) - .build(); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, oldShards, Optional.empty(), 0); - - List expectedUuids = ImmutableList.of(UUID.randomUUID(), UUID.randomUUID()); - List newShards = ImmutableList.builder() - .add(shardInfo(expectedUuids.get(0), nodes.get(0))) - .add(shardInfo(expectedUuids.get(1), nodes.get(0))) - .build(); - - Set shardMetadata = shardManager.getNodeShards(nodes.get(0)); - Set replacedUuids = shardMetadata.stream().map(ShardMetadata::getShardUuid).collect(toSet()); - - transactionId = shardManager.beginTransaction(); - shardManager.replaceShardUuids(transactionId, tableId, columns, replacedUuids, newShards, OptionalLong.of(0)); - - shardMetadata = shardManager.getNodeShards(nodes.get(0)); - Set actualUuids = shardMetadata.stream().map(ShardMetadata::getShardUuid).collect(toSet()); - assertThat(actualUuids).isEqualTo(ImmutableSet.copyOf(expectedUuids)); - - // Compute expected all uuids for this table - Set expectedAllUuids = new HashSet<>(originalUuids); - expectedAllUuids.removeAll(replacedUuids); - expectedAllUuids.addAll(expectedUuids); - - // check that shards are replaced in index table as well - Set shardNodes = ImmutableSet.copyOf(shardManager.getShardNodes(tableId, TupleDomain.all())); - Set actualAllUuids = shardNodes.stream() - .map(BucketShards::getShards) - .flatMap(Collection::stream) - .map(ShardNodes::getShardUuid) - .collect(toSet()); - assertThat(actualAllUuids).isEqualTo(expectedAllUuids); - - // verify that conflicting updates are handled - newShards = ImmutableList.of(shardInfo(UUID.randomUUID(), nodes.get(0))); - List finalNewShards = newShards; - long newTransactionId = shardManager.beginTransaction(); - assertTrinoExceptionThrownBy(() -> shardManager.replaceShardUuids(newTransactionId, tableId, columns, replacedUuids, finalNewShards, OptionalLong.of(0))) - .hasErrorCode(TRANSACTION_CONFLICT) - .hasMessage("Table was updated by a different transaction. Please retry the operation."); - } - - @Test - public void testExternalBatches() - { - long tableId = createTable("test"); - Optional externalBatchId = Optional.of("foo"); - - List shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, externalBatchId, 0); - - shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); - - List finalShards = shards; - long newTransactionId = shardManager.beginTransaction(); - assertTrinoExceptionThrownBy(() -> shardManager.commitShards(newTransactionId, tableId, columns, finalShards, externalBatchId, 0)) - .hasErrorCode(RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS) - .hasMessage("External batch already exists: foo"); - } - - @Test - public void testBucketAssignments() - { - Node node1 = createTestingNode(); - Node node2 = createTestingNode(); - Node node3 = createTestingNode(); - - TestingTicker ticker = new TestingTicker(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - int bucketCount = 13; - long distributionId = metadataDao.insertDistribution(null, "test", bucketCount); - - Set originalNodes = ImmutableSet.of(node1, node2); - ShardManager shardManager = createShardManager(dbi, () -> originalNodes, ticker); - - shardManager.createBuckets(distributionId, bucketCount); - - List assignments = shardManager.getBucketAssignments(distributionId); - assertThat(assignments.size()).isEqualTo(bucketCount); - assertThat(ImmutableSet.copyOf(assignments)).isEqualTo(nodeIds(originalNodes)); - - Set newNodes = ImmutableSet.of(node1, node3); - shardManager = createShardManager(dbi, () -> newNodes, ticker); - - ShardManager finalShardManager = shardManager; - assertTrinoExceptionThrownBy(() -> finalShardManager.getBucketAssignments(distributionId)) - .hasErrorCode(SERVER_STARTING_UP) - .hasMessage("Cannot reassign buckets while server is starting"); - - ticker.increment(2, DAYS); - assignments = shardManager.getBucketAssignments(distributionId); - assertThat(assignments.size()).isEqualTo(bucketCount); - assertThat(ImmutableSet.copyOf(assignments)).isEqualTo(nodeIds(newNodes)); - - Set singleNode = ImmutableSet.of(node1); - shardManager = createShardManager(dbi, () -> singleNode, ticker); - ticker.increment(2, DAYS); - assignments = shardManager.getBucketAssignments(distributionId); - assertThat(assignments.size()).isEqualTo(bucketCount); - assertThat(ImmutableSet.copyOf(assignments)).isEqualTo(nodeIds(singleNode)); - } - - @Test - public void testEmptyTable() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - try (ResultIterator iterator = shardManager.getShardNodes(tableId, TupleDomain.all())) { - assertThat(iterator.hasNext()).isFalse(); - } - } - - @Test - public void testEmptyTableBucketed() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, true, OptionalLong.empty()); - - try (ResultIterator iterator = shardManager.getShardNodesBucketed(tableId, true, ImmutableList.of(), TupleDomain.all())) { - assertThat(iterator.hasNext()).isFalse(); - } - } - - @Test - public void testTemporalColumnTableCreation() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, TIMESTAMP_MILLIS)); - shardManager.createTable(tableId, columns, false, OptionalLong.of(1)); - - long tableId2 = createTable("test2"); - List columns2 = ImmutableList.of(new ColumnInfo(1, TIMESTAMP_MILLIS)); - shardManager.createTable(tableId2, columns2, true, OptionalLong.of(1)); - } - - @Test - public void testShardPruning() - { - ShardInfo shard1 = shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.builder() - .add(new ColumnStats(1, 5, 10)) - .add(new ColumnStats(2, -20.0, 20.0)) - .add(new ColumnStats(3, date(2013, 5, 11), date(2013, 6, 13))) - .add(new ColumnStats(4, timestamp(2013, 5, 11, 4, 5, 6), timestamp(2013, 6, 13, 7, 8, 9))) - .add(new ColumnStats(5, "hello", "world")) - .add(new ColumnStats(6, false, true)) - .build()); - - ShardInfo shard2 = shardInfo( - UUID.randomUUID(), - "node2", - ImmutableList.builder() - .add(new ColumnStats(1, 2, 8)) - .add(new ColumnStats(2, null, 50.0)) - .add(new ColumnStats(3, date(2012, 1, 1), date(2012, 12, 31))) - .add(new ColumnStats(4, timestamp(2012, 1, 1, 2, 3, 4), timestamp(2012, 12, 31, 5, 6, 7))) - .add(new ColumnStats(5, "cat", "dog")) - .add(new ColumnStats(6, true, true)) - .build()); - - ShardInfo shard3 = shardInfo( - UUID.randomUUID(), - "node3", - ImmutableList.builder() - .add(new ColumnStats(1, 15, 20)) - .add(new ColumnStats(2, null, null)) - .add(new ColumnStats(3, date(2013, 4, 1), date(2013, 6, 1))) - .add(new ColumnStats(4, timestamp(2013, 4, 1, 8, 7, 6), timestamp(2013, 6, 1, 6, 5, 4))) - .add(new ColumnStats(5, "grape", "orange")) - .add(new ColumnStats(6, false, false)) - .build()); - - List shards = ImmutableList.builder() - .add(shard1) - .add(shard2) - .add(shard3) - .build(); - - List columns = ImmutableList.builder() - .add(new ColumnInfo(1, BIGINT)) - .add(new ColumnInfo(2, DOUBLE)) - .add(new ColumnInfo(3, DATE)) - .add(new ColumnInfo(4, TIMESTAMP_MILLIS)) - .add(new ColumnInfo(5, createVarcharType(10))) - .add(new ColumnInfo(6, BOOLEAN)) - .add(new ColumnInfo(7, VARBINARY)) - .build(); - - RaptorColumnHandle c1 = new RaptorColumnHandle("c1", 1, BIGINT); - RaptorColumnHandle c2 = new RaptorColumnHandle("c2", 2, DOUBLE); - RaptorColumnHandle c3 = new RaptorColumnHandle("c3", 3, DATE); - RaptorColumnHandle c4 = new RaptorColumnHandle("c4", 4, TIMESTAMP_MILLIS); - RaptorColumnHandle c5 = new RaptorColumnHandle("c5", 5, createVarcharType(10)); - RaptorColumnHandle c6 = new RaptorColumnHandle("c6", 6, BOOLEAN); - - long tableId = createTable("test"); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - shardAssertion(tableId).expected(shards); - - shardAssertion(tableId).equal(c1, BIGINT, 3L).expected(shard2); - shardAssertion(tableId).equal(c1, BIGINT, 8L).expected(shard1, shard2); - shardAssertion(tableId).equal(c1, BIGINT, 9L).expected(shard1); - shardAssertion(tableId).equal(c1, BIGINT, 13L).expected(); - shardAssertion(tableId).between(c1, BIGINT, 8L, 14L).expected(shard1, shard2); - shardAssertion(tableId).between(c1, BIGINT, 8L, 15L).expected(shards); - shardAssertion(tableId).between(c1, BIGINT, 8L, 16L).expected(shards); - shardAssertion(tableId).between(c1, BIGINT, 12L, 14L).expected(); - shardAssertion(tableId).between(c1, BIGINT, 5L, 10L).expected(shard1, shard2); - shardAssertion(tableId).between(c1, BIGINT, 16L, 18L).expected(shard3); - shardAssertion(tableId).between(c1, BIGINT, 1L, 25L).expected(shards); - shardAssertion(tableId).between(c1, BIGINT, 4L, 12L).expected(shard1, shard2); - shardAssertion(tableId).range(c1, lessThan(BIGINT, 5L)).expected(shard1, shard2); - shardAssertion(tableId).range(c1, lessThan(BIGINT, 4L)).expected(shard2); - shardAssertion(tableId).range(c1, lessThan(BIGINT, 11L)).expected(shard1, shard2); - shardAssertion(tableId).range(c1, lessThan(BIGINT, 25L)).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(BIGINT, 1L)).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(BIGINT, 8L)).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(BIGINT, 9L)).expected(shard1, shard3); - - shardAssertion(tableId) - .between(c1, BIGINT, -25L, 25L) - .between(c2, DOUBLE, -1000.0, 1000.0) - .between(c3, BIGINT, 0L, 50000L) - .between(c4, TIMESTAMP_MILLIS, 0L, timestamp(2015, 1, 2, 3, 4, 5)) - .between(c5, createVarcharType(10), utf8Slice("a"), utf8Slice("zzzzz")) - .between(c6, BOOLEAN, false, true) - .expected(shards); - - shardAssertion(tableId) - .between(c1, BIGINT, 4L, 12L) - .between(c3, DATE, date(2013, 3, 3), date(2013, 5, 25)) - .expected(shard1); - - shardAssertion(tableId).equal(c2, DOUBLE, 25.0).expected(shard2, shard3); - shardAssertion(tableId).equal(c2, DOUBLE, 50.1).expected(shard3); - - shardAssertion(tableId).equal(c3, DATE, date(2013, 5, 12)).expected(shard1, shard3); - - shardAssertion(tableId).range(c4, greaterThan(TIMESTAMP_MILLIS, timestamp(2013, 1, 1, 0, 0, 0))).expected(shard1, shard3); - - shardAssertion(tableId).between(c5, createVarcharType(10), utf8Slice("cow"), utf8Slice("milk")).expected(shards); - shardAssertion(tableId).equal(c5, createVarcharType(10), utf8Slice("fruit")).expected(); - shardAssertion(tableId).equal(c5, createVarcharType(10), utf8Slice("pear")).expected(shard1); - shardAssertion(tableId).equal(c5, createVarcharType(10), utf8Slice("cat")).expected(shard2); - shardAssertion(tableId).range(c5, greaterThan(createVarcharType(10), utf8Slice("gum"))).expected(shard1, shard3); - shardAssertion(tableId).range(c5, lessThan(createVarcharType(10), utf8Slice("air"))).expected(); - - shardAssertion(tableId).equal(c6, BOOLEAN, true).expected(shard1, shard2); - shardAssertion(tableId).equal(c6, BOOLEAN, false).expected(shard1, shard3); - shardAssertion(tableId).range(c6, greaterThanOrEqual(BOOLEAN, false)).expected(shards); - shardAssertion(tableId).range(c6, lessThan(BOOLEAN, true)).expected(shards); - shardAssertion(tableId).range(c6, lessThan(BOOLEAN, false)).expected(shard1, shard3); - - // Test multiple ranges - shardAssertion(tableId) - .domain(c1, createDomain(lessThan(BIGINT, 0L), greaterThan(BIGINT, 25L))) - .expected(); - - shardAssertion(tableId) - .domain(c1, createDomain(range(BIGINT, 3L, true, 4L, true), range(BIGINT, 16L, true, 18L, true))) - .expected(shard2, shard3); - - shardAssertion(tableId) - .domain(c5, createDomain( - range(createVarcharType(10), utf8Slice("gum"), true, utf8Slice("happy"), true), - range(createVarcharType(10), utf8Slice("pear"), true, utf8Slice("wall"), true))) - .expected(shard1, shard3); - - shardAssertion(tableId) - .domain(c1, createDomain(range(BIGINT, 3L, true, 4L, true), range(BIGINT, 16L, true, 18L, true))) - .domain(c5, createDomain( - range(createVarcharType(10), utf8Slice("gum"), true, utf8Slice("happy"), true), - range(createVarcharType(10), utf8Slice("pear"), true, utf8Slice("wall"), true))) - .expected(shard3); - } - - @Test - public void testShardPruningTruncatedValues() - { - String prefix = "x".repeat(MAX_BINARY_INDEX_SIZE); - - ColumnStats stats = new ColumnStats(1, prefix + "a", prefix + "z"); - ShardInfo shard = shardInfo(UUID.randomUUID(), "node", ImmutableList.of(stats)); - - List shards = ImmutableList.of(shard); - - List columns = ImmutableList.of(new ColumnInfo(1, createVarcharType(10))); - RaptorColumnHandle c1 = new RaptorColumnHandle("c1", 1, createVarcharType(10)); - - long tableId = createTable("test"); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - shardAssertion(tableId).expected(shards); - shardAssertion(tableId).equal(c1, createVarcharType(10), utf8Slice(prefix)).expected(shards); - shardAssertion(tableId).equal(c1, createVarcharType(10), utf8Slice(prefix + "c")).expected(shards); - shardAssertion(tableId).range(c1, lessThan(createVarcharType(10), utf8Slice(prefix + "c"))).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(createVarcharType(10), utf8Slice(prefix + "zzz"))).expected(shards); - - shardAssertion(tableId).between(c1, createVarcharType(10), utf8Slice("w"), utf8Slice("y")).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(createVarcharType(10), utf8Slice("x"))).expected(shards); - - shardAssertion(tableId).between(c1, createVarcharType(10), utf8Slice("x"), utf8Slice("x")).expected(); - shardAssertion(tableId).range(c1, lessThan(createVarcharType(10), utf8Slice("w"))).expected(); - shardAssertion(tableId).range(c1, lessThan(createVarcharType(10), utf8Slice("x"))).expected(); - shardAssertion(tableId).range(c1, greaterThan(createVarcharType(10), utf8Slice("y"))).expected(); - - Slice shorter = utf8Slice(prefix.substring(0, prefix.length() - 1)); - shardAssertion(tableId).equal(c1, createVarcharType(10), shorter).expected(); - shardAssertion(tableId).range(c1, lessThan(createVarcharType(10), shorter)).expected(); - shardAssertion(tableId).range(c1, greaterThan(createVarcharType(10), shorter)).expected(shards); - } - - @Test - public void testShardPruningNoStats() - { - ShardInfo shard = shardInfo(UUID.randomUUID(), "node"); - List shards = ImmutableList.of(shard); - - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - RaptorColumnHandle c1 = new RaptorColumnHandle("c1", 1, BIGINT); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - shardAssertion(tableId).expected(shards); - shardAssertion(tableId).equal(c1, BIGINT, 3L).expected(shards); - } - - @Test - public void testAddNewColumn() - throws Exception - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - int before = columnCount(tableId); - - ColumnInfo newColumn = new ColumnInfo(2, BIGINT); - shardManager.addColumn(tableId, newColumn); - int after = columnCount(tableId); - // should be 2 more: min and max columns - assertThat(after).isEqualTo(before + 2); - } - - @Test - public void testAddDuplicateColumn() - throws Exception - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - int before = columnCount(tableId); - - shardManager.addColumn(tableId, columns.get(0)); - int after = columnCount(tableId); - // no error, no columns added - assertThat(after).isEqualTo(before); - } - - @Test - public void testMaintenanceBlocked() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - Set oldShards = ImmutableSet.of(UUID.randomUUID()); - - dbi.onDemand(MetadataDao.class).blockMaintenance(tableId); - - long transactionId = shardManager.beginTransaction(); - assertTrinoExceptionThrownBy(() -> shardManager.replaceShardUuids(transactionId, tableId, columns, oldShards, ImmutableSet.of(), OptionalLong.empty())) - .hasErrorCode(TRANSACTION_CONFLICT) - .hasMessage("Maintenance is blocked for table"); - } - - private Set getShardNodes(long tableId, TupleDomain predicate) - { - try (ResultIterator iterator = shardManager.getShardNodes(tableId, predicate)) { - return ImmutableSet.copyOf(concat(transform(iterator, i -> i.getShards().iterator()))); - } - } - - private long createTable(String name) - { - return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0); - } - - public static ShardInfo shardInfo(UUID shardUuid, String nodeIdentifier) - { - return shardInfo(shardUuid, nodeIdentifier, ImmutableList.of()); - } - - public static ShardInfo shardInfo(UUID shardUuid, String nodeId, List columnStats) - { - return new ShardInfo(shardUuid, OptionalInt.empty(), ImmutableSet.of(nodeId), columnStats, 0, 0, 0, 0); - } - - private static Set toShardNodes(List shards) - { - return shards.stream() - .map(shard -> new ShardNodes(shard.getShardUuid(), shard.getNodeIdentifiers())) - .collect(toSet()); - } - - public static ShardManager createShardManager(Jdbi dbi) - { - return createShardManager(dbi, ImmutableSet::of, systemTicker()); - } - - public static ShardManager createShardManager(Jdbi dbi, NodeSupplier nodeSupplier) - { - return createShardManager(dbi, nodeSupplier, systemTicker()); - } - - public static ShardManager createShardManager(Jdbi dbi, NodeSupplier nodeSupplier, Ticker ticker) - { - DaoSupplier shardDaoSupplier = new DaoSupplier<>(dbi, H2ShardDao.class); - AssignmentLimiter assignmentLimiter = new AssignmentLimiter(nodeSupplier, ticker, new MetadataConfig()); - return new DatabaseShardManager(dbi, shardDaoSupplier, nodeSupplier, assignmentLimiter, ticker, new Duration(1, DAYS)); - } - - private static Domain createDomain(Range first, Range... ranges) - { - return Domain.create(ValueSet.ofRanges(first, ranges), false); - } - - private ShardAssertion shardAssertion(long tableId) - { - return new ShardAssertion(tableId); - } - - private class ShardAssertion - { - private final Map domains = new HashMap<>(); - private final long tableId; - - public ShardAssertion(long tableId) - { - this.tableId = tableId; - } - - public ShardAssertion domain(RaptorColumnHandle column, Domain domain) - { - domains.put(column, domain); - return this; - } - - public ShardAssertion range(RaptorColumnHandle column, Range range) - { - return domain(column, createDomain(range)); - } - - public ShardAssertion equal(RaptorColumnHandle column, Type type, Object value) - { - return domain(column, Domain.singleValue(type, value)); - } - - public ShardAssertion between(RaptorColumnHandle column, Type type, Object low, Object high) - { - return range(column, Range.range(type, low, true, high, true)); - } - - public void expected(ShardInfo... shards) - { - expected(ImmutableList.copyOf(shards)); - } - - public void expected(List shards) - { - TupleDomain predicate = TupleDomain.withColumnDomains(domains); - Set actual = getShardNodes(tableId, predicate); - assertThat(actual).isEqualTo(toShardNodes(shards)); - } - } - - private static long date(int year, int month, int day) - { - return LocalDate.of(year, month, day).toEpochDay(); - } - - private static long timestamp(int year, int month, int day, int hour, int minute, int second) - { - return ZonedDateTime.of(year, month, day, hour, minute, second, 0, UTC).toInstant().toEpochMilli(); - } - - private static Set nodeIds(Collection nodes) - { - return nodes.stream().map(Node::getNodeIdentifier).collect(toSet()); - } - - private static Node createTestingNode() - { - return new InternalNode(UUID.randomUUID().toString(), URI.create("http://test"), NodeVersion.UNKNOWN, false); - } - - private int columnCount(long tableId) - throws SQLException - { - try (Statement statement = dummyHandle.getConnection().createStatement()) { - try (ResultSet rs = statement.executeQuery(format("SELECT * FROM %s LIMIT 0", shardIndexTable(tableId)))) { - return rs.getMetaData().getColumnCount(); - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestH2DatabaseConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestH2DatabaseConfig.java deleted file mode 100644 index 8cdd7155c491..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestH2DatabaseConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestH2DatabaseConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(H2DatabaseConfig.class) - .setFilename(null)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("metadata.db.filename", "/tmp/db"); - - H2DatabaseConfig expected = new H2DatabaseConfig() - .setFilename("/tmp/db"); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataConfig.java deleted file mode 100644 index e8e3ecafc2da..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.MINUTES; - -public class TestMetadataConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(MetadataConfig.class) - .setStartupGracePeriod(new Duration(5, MINUTES)) - .setReassignmentDelay(new Duration(0, MINUTES)) - .setReassignmentInterval(new Duration(0, MINUTES))); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("raptor.startup-grace-period", "42m") - .put("raptor.reassignment-delay", "6m") - .put("raptor.reassignment-interval", "7m") - .buildOrThrow(); - - MetadataConfig expected = new MetadataConfig() - .setStartupGracePeriod(new Duration(42, MINUTES)) - .setReassignmentDelay(new Duration(6, MINUTES)) - .setReassignmentInterval(new Duration(7, MINUTES)); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataDao.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataDao.java deleted file mode 100644 index 8e6ed4e46048..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataDao.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestMetadataDao -{ - private MetadataDao dao; - private Handle dummyHandle; - - @BeforeEach - public void setup() - { - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - dao = dbi.onDemand(MetadataDao.class); - createTablesWithRetry(dbi); - } - - @AfterEach - public void tearDown() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testTemporalColumn() - { - Long columnId = 1L; - long tableId = dao.insertTable("schema1", "table1", true, false, null, 0); - dao.insertColumn(tableId, columnId, "col1", 1, "bigint", null, null); - Long temporalColumnId = dao.getTemporalColumnId(tableId); - assertThat(temporalColumnId).isNull(); - - dao.updateTemporalColumnId(tableId, columnId); - temporalColumnId = dao.getTemporalColumnId(tableId); - assertThat(temporalColumnId).isNotNull(); - assertThat(temporalColumnId).isEqualTo(columnId); - - long tableId2 = dao.insertTable("schema1", "table2", true, false, null, 0); - Long columnId2 = dao.getTemporalColumnId(tableId2); - assertThat(columnId2).isNull(); - } - - @Test - public void testGetTableInformation() - { - Long columnId = 1L; - long tableId = dao.insertTable("schema1", "table1", true, false, null, 0); - dao.insertColumn(tableId, columnId, "col1", 1, "bigint", null, null); - - Table info = dao.getTableInformation(tableId); - assertTable(info, tableId); - - info = dao.getTableInformation("schema1", "table1"); - assertTable(info, tableId); - } - - private static void assertTable(Table info, long tableId) - { - assertThat(info.getTableId()).isEqualTo(tableId); - assertThat(info.getDistributionId()).isEqualTo(Optional.empty()); - assertThat(info.getDistributionName()).isEqualTo(Optional.empty()); - assertThat(info.getBucketCount()).isEqualTo(OptionalInt.empty()); - assertThat(info.getTemporalColumnId()).isEqualTo(OptionalLong.empty()); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorMetadata.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorMetadata.java deleted file mode 100644 index 74dd0f6c3b1c..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorMetadata.java +++ /dev/null @@ -1,826 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.trino.metadata.MetadataUtil.TableMetadataBuilder; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.RaptorMetadata; -import io.trino.plugin.raptor.legacy.RaptorPartitioningHandle; -import io.trino.plugin.raptor.legacy.RaptorSessionProperties; -import io.trino.plugin.raptor.legacy.RaptorTableHandle; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.NodeManager; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableLayout; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorViewDefinition; -import io.trino.spi.connector.ConnectorViewDefinition.ViewColumn; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.testing.TestingConnectorSession; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.stream.Collectors; - -import static com.google.common.base.Ticker.systemTicker; -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.BUCKETED_ON_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.BUCKET_COUNT_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.DISTRIBUTION_NAME_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.ORDERING_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.ORGANIZED_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.spi.StandardErrorCode.TRANSACTION_CONFLICT; -import static io.trino.spi.connector.RetryMode.NO_RETRIES; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder; -import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestRaptorMetadata -{ - private static final SchemaTableName DEFAULT_TEST_ORDERS = new SchemaTableName("test", "orders"); - private static final SchemaTableName DEFAULT_TEST_LINEITEMS = new SchemaTableName("test", "lineitems"); - private static final ConnectorSession SESSION = TestingConnectorSession.builder() - .setPropertyMetadata(new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties()) - .build(); - - private Jdbi dbi; - private Handle dummyHandle; - private ShardManager shardManager; - private RaptorMetadata metadata; - - @BeforeEach - public void setupDatabase() - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - - NodeManager nodeManager = new TestingNodeManager(); - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - shardManager = createShardManager(dbi, nodeSupplier, systemTicker()); - metadata = new RaptorMetadata(dbi, shardManager); - } - - @AfterEach - public void cleanupDatabase() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testRenameColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - ColumnHandle columnHandle = metadata.getColumnHandles(SESSION, tableHandle).get("orderkey"); - - metadata.renameColumn(SESSION, raptorTableHandle, columnHandle, "orderkey_renamed"); - - assertThat(metadata.getColumnHandles(SESSION, tableHandle).get("orderkey")).isNull(); - assertThat(metadata.getColumnHandles(SESSION, tableHandle).get("orderkey_renamed")).isNotNull(); - } - - @Test - public void testAddColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, buildTable(ImmutableMap.of(), tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("price", BIGINT)), - SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - - metadata.addColumn(SESSION, raptorTableHandle, new ColumnMetadata("new_col", BIGINT)); - assertThat(metadata.getColumnHandles(SESSION, raptorTableHandle).get("new_col")).isNotNull(); - } - - @Test - public void testDropColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, buildTable(ImmutableMap.of(), tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("price", BIGINT)), - SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - - ColumnHandle lastColumn = metadata.getColumnHandles(SESSION, tableHandle).get("orderkey"); - metadata.dropColumn(SESSION, raptorTableHandle, lastColumn); - assertThat(metadata.getColumnHandles(SESSION, tableHandle).get("orderkey")).isNull(); - } - - @Test - public void testAddColumnAfterDropColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, buildTable(ImmutableMap.of(), tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("price", BIGINT)), - SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - ColumnHandle column = metadata.getColumnHandles(SESSION, tableHandle).get("orderkey"); - - metadata.dropColumn(SESSION, raptorTableHandle, column); - metadata.addColumn(SESSION, raptorTableHandle, new ColumnMetadata("new_col", BIGINT)); - assertThat(metadata.getColumnHandles(SESSION, tableHandle).get("orderkey")).isNull(); - assertThat(metadata.getColumnHandles(SESSION, raptorTableHandle).get("new_col")).isNotNull(); - } - - @Test - public void testDropColumnDisallowed() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - Map properties = ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 16, - BUCKETED_ON_PROPERTY, ImmutableList.of("orderkey"), - ORDERING_PROPERTY, ImmutableList.of("totalprice"), - TEMPORAL_COLUMN_PROPERTY, "orderdate"); - ConnectorTableMetadata ordersTable = buildTable(properties, tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("totalprice", DOUBLE) - .column("orderdate", DATE) - .column("highestid", BIGINT)); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle ordersTableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(ordersTableHandle, RaptorTableHandle.class); - RaptorTableHandle ordersRaptorTableHandle = (RaptorTableHandle) ordersTableHandle; - assertThat(ordersRaptorTableHandle.getTableId()).isEqualTo(1); - - assertInstanceOf(ordersRaptorTableHandle, RaptorTableHandle.class); - - // disallow dropping bucket, sort, temporal and highest-id columns - ColumnHandle bucketColumn = metadata.getColumnHandles(SESSION, ordersRaptorTableHandle).get("orderkey"); - assertThatThrownBy(() -> metadata.dropColumn(SESSION, ordersTableHandle, bucketColumn)) - .isInstanceOf(TrinoException.class) - .hasMessage("Cannot drop bucket columns"); - - ColumnHandle sortColumn = metadata.getColumnHandles(SESSION, ordersRaptorTableHandle).get("totalprice"); - assertThatThrownBy(() -> metadata.dropColumn(SESSION, ordersTableHandle, sortColumn)) - .isInstanceOf(TrinoException.class) - .hasMessage("Cannot drop sort columns"); - - ColumnHandle temporalColumn = metadata.getColumnHandles(SESSION, ordersRaptorTableHandle).get("orderdate"); - assertThatThrownBy(() -> metadata.dropColumn(SESSION, ordersTableHandle, temporalColumn)) - .isInstanceOf(TrinoException.class) - .hasMessage("Cannot drop the temporal column"); - - ColumnHandle highestColumn = metadata.getColumnHandles(SESSION, ordersRaptorTableHandle).get("highestid"); - assertThatThrownBy(() -> metadata.dropColumn(SESSION, ordersTableHandle, highestColumn)) - .isInstanceOf(TrinoException.class) - .hasMessage("Cannot drop the column which has the largest column ID in the table"); - } - - @Test - public void testRenameTable() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - SchemaTableName renamedTable = new SchemaTableName(raptorTableHandle.getSchemaName(), "orders_renamed"); - - metadata.renameTable(SESSION, raptorTableHandle, renamedTable); - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - ConnectorTableHandle renamedTableHandle = metadata.getTableHandle(SESSION, renamedTable, Optional.empty(), Optional.empty()); - assertThat(renamedTableHandle).isNotNull(); - assertThat(((RaptorTableHandle) renamedTableHandle).getTableName()).isEqualTo(renamedTable.getTableName()); - } - - @Test - public void testCreateTable() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - assertThat(((RaptorTableHandle) tableHandle).getTableId()).isEqualTo(1); - - ConnectorTableMetadata table = metadata.getTableMetadata(SESSION, tableHandle); - assertTableEqual(table, getOrdersTable()); - - ColumnHandle columnHandle = metadata.getColumnHandles(SESSION, tableHandle).get("orderkey"); - assertInstanceOf(columnHandle, RaptorColumnHandle.class); - assertThat(((RaptorColumnHandle) columnHandle).getColumnId()).isEqualTo(1); - - ColumnMetadata columnMetadata = metadata.getColumnMetadata(SESSION, tableHandle, columnHandle); - assertThat(columnMetadata).isNotNull(); - assertThat(columnMetadata.getName()).isEqualTo("orderkey"); - assertThat(columnMetadata.getType()).isEqualTo(BIGINT); - } - - @Test - public void testTableProperties() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of( - ORDERING_PROPERTY, ImmutableList.of("orderdate", "custkey"), - TEMPORAL_COLUMN_PROPERTY, "orderdate")); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // verify sort columns - List sortColumns = metadataDao.listSortColumns(tableId); - assertTableColumnsEqual(sortColumns, ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderdate", DATE, 4, 3, OptionalInt.empty(), OptionalInt.of(0), true), - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.empty(), OptionalInt.of(1), false))); - - // verify temporal column - assertThat(metadataDao.getTemporalColumnId(tableId)).isEqualTo(Long.valueOf(4)); - - // verify no organization - assertThat(metadataDao.getTableInformation(tableId).isOrganized()).isFalse(); - - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testTablePropertiesWithOrganization() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of( - ORDERING_PROPERTY, ImmutableList.of("orderdate", "custkey"), - ORGANIZED_PROPERTY, true)); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // verify sort columns - List sortColumns = metadataDao.listSortColumns(tableId); - assertTableColumnsEqual(sortColumns, ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderdate", DATE, 4, 3, OptionalInt.empty(), OptionalInt.of(0), false), - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.empty(), OptionalInt.of(1), false))); - - // verify organization - assertThat(metadataDao.getTableInformation(tableId).isOrganized()).isTrue(); - - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testCreateBucketedTable() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 16, - BUCKETED_ON_PROPERTY, ImmutableList.of("custkey", "orderkey"))); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - assertTableColumnsEqual(metadataDao.listBucketColumns(tableId), ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.of(0), OptionalInt.empty(), false), - new TableColumn(DEFAULT_TEST_ORDERS, "orderkey", BIGINT, 1, 0, OptionalInt.of(1), OptionalInt.empty(), false))); - - assertThat(raptorTableHandle.getBucketCount()).isEqualTo(OptionalInt.of(16)); - - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(1)); - - metadata.dropTable(SESSION, tableHandle); - - // create a new table and verify it has a different distribution - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - tableId = ((RaptorTableHandle) metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).getTableId(); - assertThat(tableId).isEqualTo(2); - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(2)); - } - - @Test - public void testCreateBucketedTableAsSelect() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 32, - BUCKETED_ON_PROPERTY, ImmutableList.of("orderkey", "custkey"))); - - ConnectorTableLayout layout = metadata.getNewTableLayout(SESSION, ordersTable).get(); - assertThat(layout.getPartitionColumns()).isEqualTo(ImmutableList.of("orderkey", "custkey")); - assertThat(layout.getPartitioning()).isPresent(); - assertInstanceOf(layout.getPartitioning().get(), RaptorPartitioningHandle.class); - RaptorPartitioningHandle partitioning = (RaptorPartitioningHandle) layout.getPartitioning().get(); - assertThat(partitioning.getDistributionId()).isEqualTo(1); - - ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, ordersTable, Optional.of(layout), NO_RETRIES, false); - metadata.finishCreateTable(SESSION, outputHandle, ImmutableList.of(), ImmutableList.of()); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - assertTableColumnsEqual(metadataDao.listBucketColumns(tableId), ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderkey", BIGINT, 1, 0, OptionalInt.of(0), OptionalInt.empty(), false), - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.of(1), OptionalInt.empty(), false))); - - assertThat(raptorTableHandle.getBucketCount()).isEqualTo(OptionalInt.of(32)); - - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(1)); - - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testCreateBucketedTableExistingDistribution() - { - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // create orders table - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata table = getOrdersTable(ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 16, - BUCKETED_ON_PROPERTY, ImmutableList.of("orderkey"), - DISTRIBUTION_NAME_PROPERTY, "orders")); - metadata.createTable(SESSION, table, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - - long tableId = raptorTableHandle.getTableId(); - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - assertTableColumnsEqual(metadataDao.listBucketColumns(tableId), ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderkey", BIGINT, 1, 0, OptionalInt.of(0), OptionalInt.empty(), false))); - - assertThat(raptorTableHandle.getBucketCount()).isEqualTo(OptionalInt.of(16)); - - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(1)); - - // create lineitems table - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_LINEITEMS, Optional.empty(), Optional.empty())).isNull(); - - table = getLineItemsTable(ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 16, - BUCKETED_ON_PROPERTY, ImmutableList.of("orderkey"), - DISTRIBUTION_NAME_PROPERTY, "orders")); - metadata.createTable(SESSION, table, SaveMode.FAIL); - - tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_LINEITEMS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - raptorTableHandle = (RaptorTableHandle) tableHandle; - - tableId = raptorTableHandle.getTableId(); - assertThat(tableId).isEqualTo(2); - - assertTableColumnsEqual(metadataDao.listBucketColumns(tableId), ImmutableList.of( - new TableColumn(DEFAULT_TEST_LINEITEMS, "orderkey", BIGINT, 1, 0, OptionalInt.of(0), OptionalInt.empty(), false))); - - assertThat(raptorTableHandle.getBucketCount()).isEqualTo(OptionalInt.of(16)); - - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(1)); - } - - @Test - public void testInvalidOrderingColumns() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of(ORDERING_PROPERTY, ImmutableList.of("orderdatefoo"))), SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Ordering column does not exist: orderdatefoo"); - } - - @Test - public void testInvalidTemporalColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "foo")), SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Temporal column does not exist: foo"); - } - - @Test - public void testInvalidTemporalColumnType() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "orderkey")), SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Temporal column must be of type timestamp or date: orderkey"); - } - - @Test - public void testInvalidTemporalOrganization() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of( - TEMPORAL_COLUMN_PROPERTY, "orderdate", - ORGANIZED_PROPERTY, true)), - SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Table with temporal columns cannot be organized"); - } - - @Test - public void testInvalidOrderingOrganization() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of(ORGANIZED_PROPERTY, true)), SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Table organization requires an ordering"); - } - - @Test - public void testSortOrderProperty() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of(ORDERING_PROPERTY, ImmutableList.of("orderdate", "custkey"))); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // verify sort columns - List sortColumns = metadataDao.listSortColumns(tableId); - assertTableColumnsEqual(sortColumns, ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderdate", DATE, 4, 3, OptionalInt.empty(), OptionalInt.of(0), false), - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.empty(), OptionalInt.of(1), false))); - - // verify temporal column is not set - assertThat(metadataDao.getTemporalColumnId(tableId)).isEqualTo(null); - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testTemporalColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "orderdate")); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // verify sort columns are not set - List sortColumns = metadataDao.listSortColumns(tableId); - assertThat(sortColumns.size()).isEqualTo(0); - assertThat(sortColumns).isEqualTo(ImmutableList.of()); - - // verify temporal column is set - assertThat(metadataDao.getTemporalColumnId(tableId)).isEqualTo(Long.valueOf(4)); - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testListTables() - { - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - List tables = metadata.listTables(SESSION, Optional.empty()); - assertThat(tables).isEqualTo(ImmutableList.of(DEFAULT_TEST_ORDERS)); - } - - @Test - public void testListTableColumns() - { - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - Map> columns = metadata.listTableColumns(SESSION, new SchemaTablePrefix()); - assertThat(columns).isEqualTo(ImmutableMap.of(DEFAULT_TEST_ORDERS, getOrdersTable().getColumns())); - } - - @Test - public void testListTableColumnsFiltering() - { - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - Map> filterCatalog = metadata.listTableColumns(SESSION, new SchemaTablePrefix()); - Map> filterSchema = metadata.listTableColumns(SESSION, new SchemaTablePrefix("test")); - Map> filterTable = metadata.listTableColumns(SESSION, new SchemaTablePrefix("test", "orders")); - assertThat(filterCatalog).isEqualTo(filterSchema); - assertThat(filterCatalog).isEqualTo(filterTable); - } - - @Test - public void testViews() - { - SchemaTableName test1 = new SchemaTableName("test", "test_view1"); - SchemaTableName test2 = new SchemaTableName("test", "test_view2"); - - // create views - metadata.createView(SESSION, test1, testingViewDefinition("test1"), ImmutableMap.of(), false); - metadata.createView(SESSION, test2, testingViewDefinition("test2"), ImmutableMap.of(), false); - - // verify listing - List list = metadata.listViews(SESSION, Optional.of("test")); - assertEqualsIgnoreOrder(list, ImmutableList.of(test1, test2)); - - // verify getting data - Map views = metadata.getViews(SESSION, Optional.of("test")); - assertThat(views.keySet()).isEqualTo(ImmutableSet.of(test1, test2)); - assertThat(views.get(test1).getOriginalSql()).isEqualTo("test1"); - assertThat(views.get(test2).getOriginalSql()).isEqualTo("test2"); - - // drop first view - metadata.dropView(SESSION, test1); - - assertThat(metadata.getViews(SESSION, Optional.of("test"))) - .containsOnlyKeys(test2); - - // drop second view - metadata.dropView(SESSION, test2); - - assertThat(metadata.getViews(SESSION, Optional.of("test"))) - .isEmpty(); - - // verify listing everything - assertThat(metadata.getViews(SESSION, Optional.empty())) - .isEmpty(); - } - - @Test - public void testCreateViewWithoutReplace() - { - SchemaTableName test = new SchemaTableName("test", "test_view"); - metadata.createView(SESSION, test, testingViewDefinition("test"), ImmutableMap.of(), false); - assertThatThrownBy(() -> metadata.createView(SESSION, test, testingViewDefinition("test"), ImmutableMap.of(), false)) - .isInstanceOf(TrinoException.class) - .hasMessage("View already exists: test.test_view"); - } - - @Test - public void testCreateViewWithReplace() - { - SchemaTableName test = new SchemaTableName("test", "test_view"); - - metadata.createView(SESSION, test, testingViewDefinition("aaa"), ImmutableMap.of(), true); - metadata.createView(SESSION, test, testingViewDefinition("bbb"), ImmutableMap.of(), true); - - assertThat(metadata.getView(SESSION, test)) - .map(ConnectorViewDefinition::getOriginalSql) - .contains("bbb"); - } - - @Test - public void testTransactionTableWrite() - { - // start table creation - long transactionId = 1; - ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, getOrdersTable(), Optional.empty(), NO_RETRIES, false); - - // transaction is in progress - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isNull(); - - // commit table creation - metadata.finishCreateTable(SESSION, outputHandle, ImmutableList.of(), ImmutableList.of()); - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isTrue(); - } - - @Test - public void testTransactionInsert() - { - // creating a table allocates a transaction - long transactionId = 1; - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - assertThat(transactionSuccessful(transactionId)).isTrue(); - - // start insert - transactionId++; - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - ConnectorInsertTableHandle insertHandle = metadata.beginInsert(SESSION, tableHandle, ImmutableList.of(), NO_RETRIES); - - // transaction is in progress - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isNull(); - - // commit insert - metadata.finishInsert(SESSION, insertHandle, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()); - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isTrue(); - } - - @Test - public void testTransactionAbort() - { - // start table creation - long transactionId = 1; - ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, getOrdersTable(), Optional.empty(), NO_RETRIES, false); - - // transaction is in progress - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isNull(); - - // force transaction to abort - shardManager.rollbackTransaction(transactionId); - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isFalse(); - - // commit table creation - assertTrinoExceptionThrownBy(() -> metadata.finishCreateTable(SESSION, outputHandle, ImmutableList.of(), ImmutableList.of())) - .hasErrorCode(TRANSACTION_CONFLICT) - .hasMessage("Transaction commit failed. Please retry the operation."); - } - - private boolean transactionExists(long transactionId) - { - return dbi.withHandle(handle -> handle - .select("SELECT count(*) FROM transactions WHERE transaction_id = ?", transactionId) - .mapTo(boolean.class) - .one()); - } - - private Boolean transactionSuccessful(long transactionId) - { - return dbi.withHandle(handle -> handle - .select("SELECT successful FROM transactions WHERE transaction_id = ?", transactionId) - .mapTo(Boolean.class) - .findFirst() - .orElse(null)); - } - - private Long getTableDistributionId(long tableId) - { - return dbi.withHandle(handle -> handle - .select("SELECT distribution_id FROM tables WHERE table_id = ?", tableId) - .mapTo(Long.class) - .findFirst() - .orElse(null)); - } - - private static ConnectorTableMetadata getOrdersTable() - { - return getOrdersTable(ImmutableMap.of()); - } - - private static ConnectorTableMetadata getOrdersTable(Map properties) - { - return buildTable(properties, tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("custkey", BIGINT) - .column("totalprice", DOUBLE) - .column("orderdate", DATE)); - } - - private static ConnectorTableMetadata getLineItemsTable(Map properties) - { - return buildTable(properties, tableMetadataBuilder(DEFAULT_TEST_LINEITEMS) - .column("orderkey", BIGINT) - .column("partkey", BIGINT) - .column("quantity", DOUBLE) - .column("price", DOUBLE)); - } - - private static ConnectorTableMetadata buildTable(Map properties, TableMetadataBuilder builder) - { - if (!properties.isEmpty()) { - for (Map.Entry entry : properties.entrySet()) { - builder.property(entry.getKey(), entry.getValue()); - } - } - return builder.build(); - } - - private static ConnectorViewDefinition testingViewDefinition(String sql) - { - return new ConnectorViewDefinition( - sql, - Optional.empty(), - Optional.empty(), - ImmutableList.of(new ViewColumn("test", BIGINT.getTypeId(), Optional.empty())), - Optional.empty(), - Optional.empty(), - true, - ImmutableList.of()); - } - - private static void assertTableEqual(ConnectorTableMetadata actual, ConnectorTableMetadata expected) - { - assertThat(actual.getTable()).isEqualTo(expected.getTable()); - - List actualColumns = actual.getColumns().stream() - .filter(columnMetadata -> !columnMetadata.isHidden()) - .collect(Collectors.toList()); - - List expectedColumns = expected.getColumns(); - assertThat(actualColumns.size()).isEqualTo(expectedColumns.size()); - for (int i = 0; i < actualColumns.size(); i++) { - ColumnMetadata actualColumn = actualColumns.get(i); - ColumnMetadata expectedColumn = expectedColumns.get(i); - assertThat(actualColumn.getName()).isEqualTo(expectedColumn.getName()); - assertThat(actualColumn.getType()).isEqualTo(expectedColumn.getType()); - } - assertThat(actual.getProperties()).isEqualTo(expected.getProperties()); - } - - private static void assertTableColumnEqual(TableColumn actual, TableColumn expected) - { - assertThat(actual.getTable()).isEqualTo(expected.getTable()); - assertThat(actual.getColumnId()).isEqualTo(expected.getColumnId()); - assertThat(actual.getColumnName()).isEqualTo(expected.getColumnName()); - assertThat(actual.getDataType()).isEqualTo(expected.getDataType()); - assertThat(actual.getOrdinalPosition()).isEqualTo(expected.getOrdinalPosition()); - assertThat(actual.getBucketOrdinal()).isEqualTo(expected.getBucketOrdinal()); - assertThat(actual.getSortOrdinal()).isEqualTo(expected.getSortOrdinal()); - assertThat(actual.isTemporal()).isEqualTo(expected.isTemporal()); - } - - private static void assertTableColumnsEqual(List actual, List expected) - { - assertThat(actual.size()).isEqualTo(expected.size()); - for (int i = 0; i < actual.size(); i++) { - assertTableColumnEqual(actual.get(i), expected.get(i)); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorSplitManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorSplitManager.java deleted file mode 100644 index 22f89a61c7bc..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorSplitManager.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.units.Duration; -import io.trino.client.NodeVersion; -import io.trino.metadata.InternalNode; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.RaptorMetadata; -import io.trino.plugin.raptor.legacy.RaptorSplitManager; -import io.trino.plugin.raptor.legacy.RaptorTableHandle; -import io.trino.plugin.raptor.legacy.RaptorTransactionHandle; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.TrinoException; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorSplitSource; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.BigintType; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static com.google.common.base.Ticker.systemTicker; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.shardInfo; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestRaptorSplitManager -{ - private static final ConnectorTableMetadata TEST_TABLE = tableMetadataBuilder(new SchemaTableName("demo", "test_table")) - .column("ds", createVarcharType(10)) - .column("foo", createVarcharType(10)) - .column("bar", BigintType.BIGINT) - .build(); - - private Handle dummyHandle; - private Path temporary; - private RaptorMetadata metadata; - private RaptorSplitManager raptorSplitManager; - private ConnectorTableHandle tableHandle; - private ShardManager shardManager; - private long tableId; - - @BeforeEach - public void setup() - throws Exception - { - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - temporary = createTempDirectory(null); - AssignmentLimiter assignmentLimiter = new AssignmentLimiter(ImmutableSet::of, systemTicker(), new MetadataConfig()); - shardManager = new DatabaseShardManager(dbi, new DaoSupplier<>(dbi, ShardDao.class), ImmutableSet::of, assignmentLimiter, systemTicker(), new Duration(0, MINUTES)); - TestingNodeManager nodeManager = new TestingNodeManager(); - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - - String nodeName = UUID.randomUUID().toString(); - nodeManager.addNode(new InternalNode(nodeName, new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false)); - - CatalogName connectorId = new CatalogName("raptor"); - metadata = new RaptorMetadata(dbi, shardManager); - - metadata.createTable(SESSION, TEST_TABLE, SaveMode.FAIL); - tableHandle = metadata.getTableHandle(SESSION, TEST_TABLE.getTable(), Optional.empty(), Optional.empty()); - - List shards = ImmutableList.builder() - .add(shardInfo(UUID.randomUUID(), nodeName)) - .add(shardInfo(UUID.randomUUID(), nodeName)) - .add(shardInfo(UUID.randomUUID(), nodeName)) - .add(shardInfo(UUID.randomUUID(), nodeName)) - .build(); - - tableId = ((RaptorTableHandle) tableHandle).getTableId(); - - List columns = metadata.getColumnHandles(SESSION, tableHandle).values().stream() - .map(RaptorColumnHandle.class::cast) - .map(ColumnInfo::fromHandle) - .collect(toList()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - raptorSplitManager = new RaptorSplitManager(connectorId, nodeSupplier, shardManager, false); - } - - @AfterEach - public void teardown() - throws IOException - { - dummyHandle.close(); - dummyHandle = null; - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testSanity() - { - ConnectorSplitSource splitSource = getSplits(raptorSplitManager, tableHandle); - int splitCount = 0; - while (!splitSource.isFinished()) { - splitCount += getSplits(splitSource, 1000).size(); - } - assertThat(splitCount).isEqualTo(4); - } - - @Test - public void testNoHostForShard() - { - assertThatThrownBy(() -> { - deleteShardNodes(); - - ConnectorSplitSource splitSource = getSplits(raptorSplitManager, tableHandle); - getSplits(splitSource, 1000); - }) - .isInstanceOf(TrinoException.class) - .hasMessageMatching("No host for shard .* found: \\[\\]"); - } - - @Test - public void testAssignRandomNodeWhenBackupAvailable() - { - TestingNodeManager nodeManager = new TestingNodeManager(); - CatalogName connectorId = new CatalogName("raptor"); - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - InternalNode node = new InternalNode(UUID.randomUUID().toString(), URI.create("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); - nodeManager.addNode(node); - RaptorSplitManager raptorSplitManagerWithBackup = new RaptorSplitManager(connectorId, nodeSupplier, shardManager, true); - - deleteShardNodes(); - - ConnectorSplitSource partitionSplit = getSplits(raptorSplitManagerWithBackup, tableHandle); - List batch = getSplits(partitionSplit, 1); - assertThat(getOnlyElement(getOnlyElement(batch).getAddresses())).isEqualTo(node.getHostAndPort()); - } - - @Test - public void testNoNodes() - { - assertThatThrownBy(() -> { - deleteShardNodes(); - - RaptorSplitManager raptorSplitManagerWithBackup = new RaptorSplitManager(new CatalogName("fbraptor"), ImmutableSet::of, shardManager, true); - ConnectorSplitSource splitSource = getSplits(raptorSplitManagerWithBackup, tableHandle); - getSplits(splitSource, 1000); - }) - .isInstanceOf(TrinoException.class) - .hasMessage("No nodes available to run query"); - } - - private void deleteShardNodes() - { - dummyHandle.execute("DELETE FROM shard_nodes"); - dummyHandle.execute(format("UPDATE %s SET node_ids = ''", shardIndexTable(tableId))); - } - - private static ConnectorSplitSource getSplits(RaptorSplitManager splitManager, ConnectorTableHandle table) - { - ConnectorTransactionHandle transaction = new RaptorTransactionHandle(); - return splitManager.getSplits(transaction, SESSION, table, DynamicFilter.EMPTY, Constraint.alwaysTrue()); - } - - private static List getSplits(ConnectorSplitSource source, int maxSize) - { - return getFutureValue(source.getNextBatch(maxSize)).getSplits(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleaner.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleaner.java deleted file mode 100644 index 8e174bc1b535..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleaner.java +++ /dev/null @@ -1,464 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableSet; -import io.airlift.testing.TestingTicker; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.backup.FileBackupStore; -import io.trino.plugin.raptor.legacy.storage.FileStorageService; -import io.trino.plugin.raptor.legacy.storage.StorageService; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.plugin.raptor.legacy.util.UuidUtil.UuidArgumentFactory; -import org.intellij.lang.annotations.Language; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.sqlobject.config.RegisterArgumentFactory; -import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder; -import static java.nio.file.Files.createTempDirectory; -import static java.util.Arrays.asList; -import static java.util.UUID.randomUUID; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardCleaner -{ - private Jdbi dbi; - private Handle dummyHandle; - private Path temporary; - private StorageService storageService; - private BackupStore backupStore; - private TestingTicker ticker; - private ShardCleaner cleaner; - - @BeforeEach - public void setup() - throws IOException - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - - temporary = createTempDirectory(null); - File directory = temporary.resolve("data").toFile(); - storageService = new FileStorageService(directory); - storageService.start(); - - File backupDirectory = temporary.resolve("backup").toFile(); - backupStore = new FileBackupStore(backupDirectory); - ((FileBackupStore) backupStore).start(); - - ticker = new TestingTicker(); - - ShardCleanerConfig config = new ShardCleanerConfig(); - cleaner = new ShardCleaner( - new DaoSupplier<>(dbi, H2ShardDao.class), - "node1", - true, - ticker, - storageService, - Optional.of(backupStore), - config.getMaxTransactionAge(), - config.getTransactionCleanerInterval(), - config.getLocalCleanerInterval(), - config.getLocalCleanTime(), - config.getBackupCleanerInterval(), - config.getBackupCleanTime(), - config.getBackupDeletionThreads(), - config.getMaxCompletedTransactionAge()); - } - - @AfterEach - public void teardown() - throws IOException - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testAbortOldTransactions() - { - TestingDao dao = dbi.onDemand(TestingDao.class); - - long now = System.currentTimeMillis(); - - long txn1 = dao.insertTransaction(new Timestamp(now - HOURS.toMillis(26))); - long txn2 = dao.insertTransaction(new Timestamp(now - HOURS.toMillis(25))); - long txn3 = dao.insertTransaction(new Timestamp(now)); - - ShardDao shardDao = dbi.onDemand(ShardDao.class); - assertThat(shardDao.finalizeTransaction(txn1, true)).isEqualTo(1); - - assertQuery("SELECT transaction_id, successful FROM transactions", - row(txn1, true), - row(txn2, null), - row(txn3, null)); - - cleaner.abortOldTransactions(); - - assertQuery("SELECT transaction_id, successful FROM transactions", - row(txn1, true), - row(txn2, false), - row(txn3, null)); - } - - @Test - public void testDeleteOldShards() - { - assertThat(cleaner.getBackupShardsQueued().getTotalCount()).isEqualTo(0); - - ShardDao dao = dbi.onDemand(ShardDao.class); - - UUID shard1 = randomUUID(); - UUID shard2 = randomUUID(); - UUID shard3 = randomUUID(); - - // shards for failed transaction - long txn1 = dao.insertTransaction(); - assertThat(dao.finalizeTransaction(txn1, false)).isEqualTo(1); - - dao.insertCreatedShard(shard1, txn1); - dao.insertCreatedShard(shard2, txn1); - - // shards for running transaction - long txn2 = dao.insertTransaction(); - - dao.insertCreatedShard(shard3, txn2); - - // verify database - assertQuery("SELECT shard_uuid, transaction_id FROM created_shards", - row(shard1, txn1), - row(shard2, txn1), - row(shard3, txn2)); - - assertQuery("SELECT shard_uuid FROM deleted_shards"); - - // move shards for failed transaction to deleted - cleaner.deleteOldShards(); - - assertThat(cleaner.getBackupShardsQueued().getTotalCount()).isEqualTo(2); - - // verify database - assertQuery("SELECT shard_uuid, transaction_id FROM created_shards", - row(shard3, txn2)); - - assertQuery("SELECT shard_uuid FROM deleted_shards", - row(shard1), - row(shard2)); - } - - @Test - public void testCleanLocalShardsImmediately() - throws Exception - { - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(0); - TestingShardDao shardDao = dbi.onDemand(TestingShardDao.class); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - long tableId = metadataDao.insertTable("test", "test", false, false, null, 0); - - UUID shard1 = randomUUID(); - UUID shard2 = randomUUID(); - UUID shard3 = randomUUID(); - - Set shards = ImmutableSet.of(shard1, shard2, shard3); - - for (UUID shard : shards) { - shardDao.insertShard(shard, tableId, null, 0, 0, 0, 0); - createShardFile(shard); - assertThat(shardFileExists(shard)).isTrue(); - } - - int node1 = shardDao.insertNode("node1"); - int node2 = shardDao.insertNode("node2"); - - // shard 1: referenced by this node - // shard 2: not referenced - // shard 3: referenced by other node - - shardDao.insertShardNode(shard1, node1); - shardDao.insertShardNode(shard3, node2); - - // clean shards immediately - Set local = cleaner.getLocalShards(); - cleaner.cleanLocalShardsImmediately(local); - - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(2); - - // shards 2 and 3 should be deleted - // shard 1 is referenced by this node - assertThat(shardFileExists(shard1)).isTrue(); - assertThat(shardFileExists(shard2)).isFalse(); - assertThat(shardFileExists(shard3)).isFalse(); - } - - @Test - public void testCleanLocalShards() - throws Exception - { - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(0); - - TestingShardDao shardDao = dbi.onDemand(TestingShardDao.class); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - long tableId = metadataDao.insertTable("test", "test", false, false, null, 0); - - UUID shard1 = randomUUID(); - UUID shard2 = randomUUID(); - UUID shard3 = randomUUID(); - UUID shard4 = randomUUID(); - - Set shards = ImmutableSet.of(shard1, shard2, shard3, shard4); - - for (UUID shard : shards) { - shardDao.insertShard(shard, tableId, null, 0, 0, 0, 0); - createShardFile(shard); - assertThat(shardFileExists(shard)).isTrue(); - } - - int node1 = shardDao.insertNode("node1"); - int node2 = shardDao.insertNode("node2"); - - // shard 1: referenced by this node - // shard 2: not referenced - // shard 3: not referenced - // shard 4: referenced by other node - - shardDao.insertShardNode(shard1, node1); - shardDao.insertShardNode(shard4, node2); - - // mark unreferenced shards - cleaner.cleanLocalShards(); - - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(0); - - // make sure nothing is deleted - for (UUID shard : shards) { - assertThat(shardFileExists(shard)).isTrue(); - } - - // add reference for shard 3 - shardDao.insertShardNode(shard3, node1); - - // advance time beyond clean time - Duration cleanTime = new ShardCleanerConfig().getLocalCleanTime(); - ticker.increment(cleanTime.toMillis() + 1, MILLISECONDS); - - // clean shards - cleaner.cleanLocalShards(); - - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(2); - - // shards 2 and 4 should be deleted - // shards 1 and 3 are referenced by this node - assertThat(shardFileExists(shard1)).isTrue(); - assertThat(shardFileExists(shard2)).isFalse(); - assertThat(shardFileExists(shard3)).isTrue(); - assertThat(shardFileExists(shard4)).isFalse(); - } - - @Test - public void testCleanBackupShards() - throws Exception - { - assertThat(cleaner.getBackupShardsCleaned().getTotalCount()).isEqualTo(0); - - TestingDao dao = dbi.onDemand(TestingDao.class); - - UUID shard1 = randomUUID(); - UUID shard2 = randomUUID(); - UUID shard3 = randomUUID(); - - long now = System.currentTimeMillis(); - Timestamp time1 = new Timestamp(now - HOURS.toMillis(25)); - Timestamp time2 = new Timestamp(now - HOURS.toMillis(23)); - - // shard 1: should be cleaned - dao.insertDeletedShard(shard1, time1); - - // shard 2: should be cleaned - dao.insertDeletedShard(shard2, time1); - - // shard 3: deleted too recently - dao.insertDeletedShard(shard3, time2); - - createShardBackups(shard1, shard2, shard3); - - cleaner.cleanBackupShards(); - - assertThat(cleaner.getBackupShardsCleaned().getTotalCount()).isEqualTo(2); - - assertThat(shardBackupExists(shard1)).isFalse(); - assertThat(shardBackupExists(shard2)).isFalse(); - assertThat(shardBackupExists(shard3)).isTrue(); - - assertQuery("SELECT shard_uuid FROM deleted_shards", - row(shard3)); - } - - @Test - public void testDeleteOldCompletedTransactions() - { - TestingDao dao = dbi.onDemand(TestingDao.class); - ShardDao shardDao = dbi.onDemand(ShardDao.class); - - long now = System.currentTimeMillis(); - Timestamp yesterdayStart = new Timestamp(now - HOURS.toMillis(27)); - Timestamp yesterdayEnd = new Timestamp(now - HOURS.toMillis(26)); - Timestamp todayEnd = new Timestamp(now - HOURS.toMillis(1)); - - long txn1 = dao.insertTransaction(yesterdayStart); - long txn2 = dao.insertTransaction(yesterdayStart); - long txn3 = dao.insertTransaction(yesterdayStart); - long txn4 = dao.insertTransaction(yesterdayStart); - long txn5 = dao.insertTransaction(new Timestamp(now)); - long txn6 = dao.insertTransaction(new Timestamp(now)); - - assertThat(shardDao.finalizeTransaction(txn1, true)).isEqualTo(1); - assertThat(shardDao.finalizeTransaction(txn2, false)).isEqualTo(1); - assertThat(shardDao.finalizeTransaction(txn3, false)).isEqualTo(1); - assertThat(shardDao.finalizeTransaction(txn5, true)).isEqualTo(1); - assertThat(shardDao.finalizeTransaction(txn6, false)).isEqualTo(1); - - assertThat(dao.updateTransactionEndTime(txn1, yesterdayEnd)).isEqualTo(1); - assertThat(dao.updateTransactionEndTime(txn2, yesterdayEnd)).isEqualTo(1); - assertThat(dao.updateTransactionEndTime(txn3, yesterdayEnd)).isEqualTo(1); - assertThat(dao.updateTransactionEndTime(txn5, todayEnd)).isEqualTo(1); - assertThat(dao.updateTransactionEndTime(txn6, todayEnd)).isEqualTo(1); - - shardDao.insertCreatedShard(randomUUID(), txn2); - shardDao.insertCreatedShard(randomUUID(), txn2); - - assertQuery("SELECT transaction_id, successful, end_time FROM transactions", - row(txn1, true, yesterdayEnd), // old successful - row(txn2, false, yesterdayEnd), // old failed, shards present - row(txn3, false, yesterdayEnd), // old failed, no referencing shards - row(txn4, null, null), // old not finished - row(txn5, true, todayEnd), // new successful - row(txn6, false, todayEnd)); // new failed, no referencing shards - - cleaner.deleteOldCompletedTransactions(); - - assertQuery("SELECT transaction_id, successful, end_time FROM transactions", - row(txn2, false, yesterdayEnd), - row(txn4, null, null), - row(txn5, true, todayEnd), - row(txn6, false, todayEnd)); - } - - private boolean shardFileExists(UUID uuid) - { - return storageService.getStorageFile(uuid).exists(); - } - - private void createShardFile(UUID uuid) - throws IOException - { - File file = storageService.getStorageFile(uuid); - storageService.createParents(file); - assertThat(file.createNewFile()).isTrue(); - } - - private boolean shardBackupExists(UUID uuid) - { - return backupStore.shardExists(uuid); - } - - private void createShardBackups(UUID... uuids) - throws IOException - { - for (UUID uuid : uuids) { - File file = temporary.resolve("empty-" + randomUUID()).toFile(); - assertThat(file.createNewFile()).isTrue(); - backupStore.backupShard(uuid, file); - } - } - - @SafeVarargs - private void assertQuery(@Language("SQL") String sql, List... rows) - { - assertEqualsIgnoreOrder(select(sql), asList(rows)); - } - - private List> select(@Language("SQL") String sql) - { - return dbi.withHandle(handle -> handle.createQuery(sql) - .map((rs, index, context) -> { - int count = rs.getMetaData().getColumnCount(); - List row = new ArrayList<>(count); - for (int i = 1; i <= count; i++) { - Object value = rs.getObject(i); - if (value instanceof byte[]) { - value = uuidFromBytes((byte[]) value); - } - row.add(value); - } - return row; - }) - .list()); - } - - private static List row(Object... values) - { - return asList(values); - } - - @RegisterArgumentFactory(UuidArgumentFactory.class) - private interface TestingDao - { - @SqlUpdate("INSERT INTO transactions (start_time) VALUES (:startTime)") - @GetGeneratedKeys - long insertTransaction(Timestamp startTime); - - @SqlUpdate("INSERT INTO deleted_shards (shard_uuid, delete_time)\n" + - "VALUES (:shardUuid, :deleteTime)") - void insertDeletedShard(UUID shardUuid, Timestamp deleteTime); - - @SqlUpdate("UPDATE transactions SET end_time = :endTime WHERE transaction_id = :transactionId") - int updateTransactionEndTime(long transactionId, Timestamp endTime); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleanerConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleanerConfig.java deleted file mode 100644 index 92fff624f373..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleanerConfig.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MINUTES; - -public class TestShardCleanerConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(ShardCleanerConfig.class) - .setMaxTransactionAge(new Duration(1, DAYS)) - .setTransactionCleanerInterval(new Duration(10, MINUTES)) - .setLocalCleanerInterval(new Duration(1, HOURS)) - .setLocalCleanTime(new Duration(4, HOURS)) - .setBackupCleanerInterval(new Duration(5, MINUTES)) - .setBackupCleanTime(new Duration(1, DAYS)) - .setBackupDeletionThreads(50) - .setMaxCompletedTransactionAge(new Duration(1, DAYS))); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("raptor.max-transaction-age", "42m") - .put("raptor.transaction-cleaner-interval", "43m") - .put("raptor.local-cleaner-interval", "31m") - .put("raptor.local-clean-time", "32m") - .put("raptor.backup-cleaner-interval", "34m") - .put("raptor.backup-clean-time", "35m") - .put("raptor.backup-deletion-threads", "37") - .put("raptor.max-completed-transaction-age", "39m") - .buildOrThrow(); - - ShardCleanerConfig expected = new ShardCleanerConfig() - .setMaxTransactionAge(new Duration(42, MINUTES)) - .setTransactionCleanerInterval(new Duration(43, MINUTES)) - .setLocalCleanerInterval(new Duration(31, MINUTES)) - .setLocalCleanTime(new Duration(32, MINUTES)) - .setBackupCleanerInterval(new Duration(34, MINUTES)) - .setBackupCleanTime(new Duration(35, MINUTES)) - .setBackupDeletionThreads(37) - .setMaxCompletedTransactionAge(new Duration(39, MINUTES)); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardDao.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardDao.java deleted file mode 100644 index 09751ea27780..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardDao.java +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.statement.UnableToExecuteStatementException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.sql.SQLException; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardDao -{ - private TestingShardDao dao; - private Jdbi dbi; - private Handle dummyHandle; - - @BeforeEach - public void setup() - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - dao = dbi.onDemand(TestingShardDao.class); - createTablesWithRetry(dbi); - } - - @AfterEach - public void teardown() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testExternalBatches() - { - assertThat(dao.externalBatchExists("foo")).isFalse(); - assertThat(dao.externalBatchExists("bar")).isFalse(); - - dao.insertExternalBatch("foo"); - - assertThat(dao.externalBatchExists("foo")).isTrue(); - assertThat(dao.externalBatchExists("bar")).isFalse(); - - assertThatThrownBy(() -> dao.insertExternalBatch("foo")) - .isInstanceOfSatisfying(UnableToExecuteStatementException.class, e -> { - assertInstanceOf(e.getCause(), SQLException.class); - assertThat(((SQLException) e.getCause()).getSQLState()).startsWith("23"); - }); - } - - @Test - public void testInsertCreatedShard() - { - long transactionId = dao.insertTransaction(); - dao.insertCreatedShard(UUID.randomUUID(), transactionId); - dao.deleteCreatedShards(transactionId); - } - - @Test - public void testInsertDeletedShards() - { - dao.insertDeletedShards(ImmutableList.of(UUID.randomUUID(), UUID.randomUUID())); - dao.insertDeletedShards(0); - } - - @Test - public void testNodeInsert() - { - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of()); - - String nodeName = UUID.randomUUID().toString(); - int nodeId = dao.insertNode(nodeName); - assertThat(dao.getNodeId(nodeName)).isEqualTo((Integer) nodeId); - - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of(nodeName)); - } - - @Test - public void testInsertShard() - { - long tableId = createTable("test"); - UUID shardUuid = UUID.randomUUID(); - long shardId = dao.insertShard(shardUuid, tableId, null, 13, 42, 84, 1234); - - ShardMetadata shard = dao.getShard(shardUuid); - assertThat(shard).isNotNull(); - assertThat(shard.getTableId()).isEqualTo(tableId); - assertThat(shard.getShardId()).isEqualTo(shardId); - assertThat(shard.getShardUuid()).isEqualTo(shardUuid); - assertThat(shard.getRowCount()).isEqualTo(13); - assertThat(shard.getCompressedSize()).isEqualTo(42); - assertThat(shard.getUncompressedSize()).isEqualTo(84); - assertThat(shard.getXxhash64()).isEqualTo(OptionalLong.of(1234)); - } - - @Test - public void testInsertShardNodeUsingShardUuid() - { - int nodeId = dao.insertNode("node"); - - long tableId = createTable("test"); - UUID shard = UUID.randomUUID(); - dao.insertShard(shard, tableId, null, 0, 0, 0, 0); - - dao.insertShardNode(shard, nodeId); - - assertThat(dao.getShardNodes(tableId)).containsExactlyElementsOf(ImmutableList.of(new ShardNode(shard, "node"))); - } - - @Test - public void testNodeShards() - { - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of()); - - String nodeName1 = UUID.randomUUID().toString(); - int nodeId1 = dao.insertNode(nodeName1); - - String nodeName2 = UUID.randomUUID().toString(); - int nodeId2 = dao.insertNode(nodeName2); - - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of(nodeName1, nodeName2)); - - UUID shardUuid1 = UUID.randomUUID(); - UUID shardUuid2 = UUID.randomUUID(); - UUID shardUuid3 = UUID.randomUUID(); - UUID shardUuid4 = UUID.randomUUID(); - UUID shardUuid5 = UUID.randomUUID(); - - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - int bucketCount = 20; - long distributionId = metadataDao.insertDistribution("test", "bigint", bucketCount); - for (int i = 0; i < bucketCount; i++) { - Integer nodeId = ((i % 2) == 0) ? nodeId1 : nodeId2; - dao.insertBuckets(distributionId, ImmutableList.of(i), ImmutableList.of(nodeId)); - } - - long plainTableId = metadataDao.insertTable("test", "plain", false, false, null, 0); - long bucketedTableId = metadataDao.insertTable("test", "bucketed", false, false, distributionId, 0); - - long shardId1 = dao.insertShard(shardUuid1, plainTableId, null, 1, 11, 111, 888_111); - long shardId2 = dao.insertShard(shardUuid2, plainTableId, null, 2, 22, 222, 888_222); - long shardId3 = dao.insertShard(shardUuid3, bucketedTableId, 8, 3, 33, 333, 888_333); - long shardId4 = dao.insertShard(shardUuid4, bucketedTableId, 9, 4, 44, 444, 888_444); - long shardId5 = dao.insertShard(shardUuid5, bucketedTableId, 7, 5, 55, 555, 888_555); - - OptionalInt noBucket = OptionalInt.empty(); - OptionalLong noRange = OptionalLong.empty(); - ShardMetadata shard1 = new ShardMetadata(plainTableId, shardId1, shardUuid1, noBucket, 1, 11, 111, OptionalLong.of(888_111), noRange, noRange); - ShardMetadata shard2 = new ShardMetadata(plainTableId, shardId2, shardUuid2, noBucket, 2, 22, 222, OptionalLong.of(888_222), noRange, noRange); - ShardMetadata shard3 = new ShardMetadata(bucketedTableId, shardId3, shardUuid3, OptionalInt.of(8), 3, 33, 333, OptionalLong.of(888_333), noRange, noRange); - ShardMetadata shard4 = new ShardMetadata(bucketedTableId, shardId4, shardUuid4, OptionalInt.of(9), 4, 44, 444, OptionalLong.of(888_444), noRange, noRange); - ShardMetadata shard5 = new ShardMetadata(bucketedTableId, shardId5, shardUuid5, OptionalInt.of(7), 5, 55, 555, OptionalLong.of(888_555), noRange, noRange); - - assertThat(dao.getShards(plainTableId)).isEqualTo(ImmutableSet.of(shardUuid1, shardUuid2)); - assertThat(dao.getShards(bucketedTableId)).isEqualTo(ImmutableSet.of(shardUuid3, shardUuid4, shardUuid5)); - - assertThat(dao.getNodeShards(nodeName1, null)).isEqualTo(ImmutableSet.of(shard3)); - assertThat(dao.getNodeShards(nodeName2, null)).isEqualTo(ImmutableSet.of(shard4, shard5)); - assertThat(dao.getNodeSizes()).isEqualTo(ImmutableSet.of( - new NodeSize(nodeName1, 33), - new NodeSize(nodeName2, 44 + 55))); - - dao.insertShardNode(shardId1, nodeId1); - dao.insertShardNode(shardId2, nodeId1); - dao.insertShardNode(shardId1, nodeId2); - - assertThat(dao.getNodeShards(nodeName1, null)).isEqualTo(ImmutableSet.of(shard1, shard2, shard3)); - assertThat(dao.getNodeShards(nodeName2, null)).isEqualTo(ImmutableSet.of(shard1, shard4, shard5)); - assertThat(dao.getNodeSizes()).isEqualTo(ImmutableSet.of( - new NodeSize(nodeName1, 11 + 22 + 33), - new NodeSize(nodeName2, 11 + 44 + 55))); - - dao.dropShardNodes(plainTableId); - - assertThat(dao.getNodeShards(nodeName1, null)).isEqualTo(ImmutableSet.of(shard3)); - assertThat(dao.getNodeShards(nodeName2, null)).isEqualTo(ImmutableSet.of(shard4, shard5)); - assertThat(dao.getNodeSizes()).isEqualTo(ImmutableSet.of( - new NodeSize(nodeName1, 33), - new NodeSize(nodeName2, 44 + 55))); - - dao.dropShards(plainTableId); - dao.dropShards(bucketedTableId); - - assertThat(dao.getShards(plainTableId)).isEqualTo(ImmutableSet.of()); - assertThat(dao.getShards(bucketedTableId)).isEqualTo(ImmutableSet.of()); - assertThat(dao.getNodeSizes()).isEqualTo(ImmutableSet.of()); - } - - @Test - public void testShardSelection() - { - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of()); - - String nodeName1 = UUID.randomUUID().toString(); - int nodeId1 = dao.insertNode(nodeName1); - - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of(nodeName1)); - - String nodeName2 = UUID.randomUUID().toString(); - int nodeId2 = dao.insertNode(nodeName2); - - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of(nodeName1, nodeName2)); - - long tableId = createTable("test"); - - UUID shardUuid1 = UUID.randomUUID(); - UUID shardUuid2 = UUID.randomUUID(); - UUID shardUuid3 = UUID.randomUUID(); - UUID shardUuid4 = UUID.randomUUID(); - - long shardId1 = dao.insertShard(shardUuid1, tableId, null, 0, 0, 0, 0); - long shardId2 = dao.insertShard(shardUuid2, tableId, null, 0, 0, 0, 0); - long shardId3 = dao.insertShard(shardUuid3, tableId, null, 0, 0, 0, 0); - long shardId4 = dao.insertShard(shardUuid4, tableId, null, 0, 0, 0, 0); - - Set shards = dao.getShards(tableId); - assertThat(shards.size()).isEqualTo(4); - - assertThat(shards).contains(shardUuid1); - assertThat(shards).contains(shardUuid2); - assertThat(shards).contains(shardUuid3); - assertThat(shards).contains(shardUuid4); - - assertThat(dao.getShardNodes(tableId).size()).isEqualTo(0); - - dao.insertShardNode(shardId1, nodeId1); - dao.insertShardNode(shardId1, nodeId2); - dao.insertShardNode(shardId2, nodeId1); - dao.insertShardNode(shardId3, nodeId1); - dao.insertShardNode(shardId4, nodeId1); - dao.insertShardNode(shardId4, nodeId2); - - assertThat(dao.getShards(tableId)).isEqualTo(shards); - - Set shardNodes = dao.getShardNodes(tableId); - assertThat(shardNodes.size()).isEqualTo(6); - - assertContainsShardNode(shardNodes, nodeName1, shardUuid1); - assertContainsShardNode(shardNodes, nodeName2, shardUuid1); - assertContainsShardNode(shardNodes, nodeName1, shardUuid2); - assertContainsShardNode(shardNodes, nodeName1, shardUuid3); - assertContainsShardNode(shardNodes, nodeName1, shardUuid4); - assertContainsShardNode(shardNodes, nodeName2, shardUuid4); - } - - private long createTable(String name) - { - return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0); - } - - private static void assertContainsShardNode(Set nodes, String nodeName, UUID shardUuid) - { - assertThat(nodes).contains(new ShardNode(shardUuid, nodeName)); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardPredicate.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardPredicate.java deleted file mode 100644 index c7cc79e6cac9..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardPredicate.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.predicate.SortedRangeSet; -import io.trino.spi.predicate.TupleDomain; -import org.junit.jupiter.api.Test; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.bucketNumberColumnHandle; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.shardUuidColumnHandle; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidStringToBytes; -import static io.trino.spi.predicate.Domain.create; -import static io.trino.spi.predicate.Domain.singleValue; -import static io.trino.spi.predicate.Range.equal; -import static io.trino.spi.predicate.Range.greaterThanOrEqual; -import static io.trino.spi.predicate.TupleDomain.withColumnDomains; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.sql.JDBCType.VARBINARY; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestShardPredicate -{ - @Test - public void testSimpleShardUuidPredicate() - { - String uuid = randomUUID().toString(); - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - shardUuidColumnHandle(), singleValue(VARCHAR, utf8Slice(uuid)))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - - assertThat(shardPredicate.getPredicate()).isEqualTo("shard_uuid = ?"); - assertThat(shardPredicate.getTypes()).isEqualTo(ImmutableList.of(VARBINARY)); - assertThat(shardPredicate.getValues()).isEqualTo(ImmutableList.of(uuidStringToBytes(utf8Slice(uuid)))); - } - - @Test - public void testDiscreteShardUuidPredicate() - { - Slice uuid0 = utf8Slice(randomUUID().toString()); - Slice uuid1 = utf8Slice(randomUUID().toString()); - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - shardUuidColumnHandle(), - create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(equal(VARCHAR, uuid0), equal(VARCHAR, uuid1))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - - assertThat(shardPredicate.getPredicate()).isEqualTo("shard_uuid = ? OR shard_uuid = ?"); - assertThat(shardPredicate.getTypes()).isEqualTo(ImmutableList.of(VARBINARY, VARBINARY)); - assertThat(ImmutableSet.copyOf(shardPredicate.getValues())).isEqualTo(ImmutableSet.of(uuidStringToBytes(uuid0), uuidStringToBytes(uuid1))); - } - - @Test - public void testInvalidUuid() - { - Slice uuid0 = utf8Slice("test1"); - Slice uuid1 = utf8Slice("test2"); - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - shardUuidColumnHandle(), - create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(equal(VARCHAR, uuid0), equal(VARCHAR, uuid1))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - - assertThat(shardPredicate.getPredicate()).isEqualTo("true"); - } - - @Test - public void testRangeShardUuidPredicate() - { - Slice uuid0 = utf8Slice(randomUUID().toString()); - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - shardUuidColumnHandle(), - create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(greaterThanOrEqual(VARCHAR, uuid0))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - assertThat(shardPredicate.getPredicate()).isEqualTo("true"); - } - - @Test - public void testBucketNumberSingleRange() - { - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - bucketNumberColumnHandle(), - create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - assertThat(shardPredicate.getPredicate()).isEqualTo("(((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)))"); - } - - @Test - public void testBucketNumberMultipleRanges() - { - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - bucketNumberColumnHandle(), - create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - assertThat(shardPredicate.getPredicate()).isEqualTo("(((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL))" + - " OR ((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)))"); - } - - @Test - public void testMultipleColumnsMultipleRanges() - { - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - bucketNumberColumnHandle(), - create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false), - new RaptorColumnHandle("col", 1, INTEGER), - create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false))); - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - assertThat(shardPredicate.getPredicate()).isEqualTo("(((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)) " + - "OR ((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL))) " + - "AND (((c1_max >= ? OR c1_max IS NULL) AND (c1_min <= ? OR c1_min IS NULL)) " + - "OR ((c1_max >= ? OR c1_max IS NULL) AND (c1_min <= ? OR c1_min IS NULL)))"); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestingShardDao.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestingShardDao.java deleted file mode 100644 index c402797352b6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestingShardDao.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; -import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.util.Set; -import java.util.UUID; - -@RegisterConstructorMapper(ShardNode.class) -interface TestingShardDao - extends H2ShardDao -{ - @SqlQuery("SELECT shard_uuid FROM shards WHERE table_id = :tableId") - Set getShards(long tableId); - - @SqlQuery("SELECT s.shard_uuid, n.node_identifier\n" + - "FROM shards s\n" + - "JOIN shard_nodes sn ON (s.shard_id = sn.shard_id)\n" + - "JOIN nodes n ON (sn.node_id = n.node_id)\n" + - "WHERE s.table_id = :tableId") - Set getShardNodes(long tableId); - - @SqlQuery("SELECT node_identifier FROM nodes") - Set getAllNodesInUse(); - - @SqlUpdate("INSERT INTO shards (shard_uuid, table_id, bucket_number, create_time, row_count, compressed_size, uncompressed_size, xxhash64)\n" + - "VALUES (:shardUuid, :tableId, :bucketNumber, CURRENT_TIMESTAMP, :rowCount, :compressedSize, :uncompressedSize, :xxhash64)") - @GetGeneratedKeys - long insertShard( - UUID shardUuid, - long tableId, - Integer bucketNumber, - long rowCount, - long compressedSize, - long uncompressedSize, - long xxhash64); - - @SqlUpdate("INSERT INTO shard_nodes (shard_id, node_id)\n" + - "VALUES (:shardId, :nodeId)\n") - void insertShardNode(long shardId, int nodeId); -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorFileBasedSecurity.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorFileBasedSecurity.java deleted file mode 100644 index dade69191d45..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorFileBasedSecurity.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import com.google.common.io.Resources; -import io.trino.Session; -import io.trino.plugin.raptor.legacy.RaptorQueryRunner; -import io.trino.spi.security.Identity; -import io.trino.testing.QueryRunner; -import io.trino.tpch.TpchTable; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; - -import static io.trino.testing.TestingSession.testSessionBuilder; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; - -@TestInstance(PER_CLASS) -@Execution(CONCURRENT) -public class TestRaptorFileBasedSecurity -{ - private final QueryRunner queryRunner; - - public TestRaptorFileBasedSecurity() - throws Exception - { - String path = new File(Resources.getResource(getClass(), "security.json").toURI()).getPath(); - queryRunner = RaptorQueryRunner.builder() - .addConnectorProperty("security.config-file", path) - .addConnectorProperty("raptor.security", "file") - .setInitialTables(TpchTable.getTables()) - .build(); - } - - @AfterAll - public void tearDown() - { - queryRunner.close(); - } - - @Test - public void testAdminCanRead() - { - Session admin = getSession("user"); - queryRunner.execute(admin, "SELECT * FROM orders"); - } - - @Test - public void testNonAdminCannotRead() - { - assertThatThrownBy(() -> { - Session bob = getSession("bob"); - queryRunner.execute(bob, "SELECT * FROM orders"); - }) - .isInstanceOf(RuntimeException.class) - .hasMessageMatching(".*Access Denied: Cannot select from table tpch.orders.*"); - } - - private Session getSession(String user) - { - return testSessionBuilder() - .setCatalog(queryRunner.getDefaultSession().getCatalog()) - .setSchema(queryRunner.getDefaultSession().getSchema()) - .setIdentity(Identity.ofUser(user)) - .build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorReadOnlySecurity.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorReadOnlySecurity.java deleted file mode 100644 index 65e647a8ba40..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorReadOnlySecurity.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import io.trino.plugin.raptor.legacy.RaptorQueryRunner; -import io.trino.testing.QueryRunner; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class TestRaptorReadOnlySecurity -{ - @Test - public void testCannotWrite() - throws Exception - { - try (QueryRunner queryRunner = RaptorQueryRunner.builder().addConnectorProperty("raptor.security", "read-only").build()) { - assertThatThrownBy(() -> { - queryRunner.execute("CREATE TABLE test_create (a bigint, b double, c varchar)"); - }) - .isInstanceOf(RuntimeException.class) - .hasMessageMatching(".*Access Denied: Cannot create .*"); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorSecurityConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorSecurityConfig.java deleted file mode 100644 index 449067454b2f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorSecurityConfig.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static io.trino.plugin.raptor.legacy.security.RaptorSecurity.ALLOW_ALL; -import static io.trino.plugin.raptor.legacy.security.RaptorSecurity.READ_ONLY; - -public class TestRaptorSecurityConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(RaptorSecurityConfig.class) - .setSecuritySystem(ALLOW_ALL)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("raptor.security", "read-only"); - - RaptorSecurityConfig expected = new RaptorSecurityConfig() - .setSecuritySystem(READ_ONLY); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/InMemoryShardRecorder.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/InMemoryShardRecorder.java deleted file mode 100644 index 85fac0528dcf..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/InMemoryShardRecorder.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -import static java.util.Objects.requireNonNull; - -public class InMemoryShardRecorder - implements ShardRecorder -{ - private final List shards = new ArrayList<>(); - - public List getShards() - { - return ImmutableList.copyOf(shards); - } - - @Override - public void recordCreatedShard(long transactionId, UUID shardUuid) - { - shards.add(new RecordedShard(transactionId, shardUuid)); - } - - public static class RecordedShard - { - private final long transactionId; - private final UUID shardUuid; - - public RecordedShard(long transactionId, UUID shardUuid) - { - this.transactionId = transactionId; - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - } - - public long getTransactionId() - { - return transactionId; - } - - public UUID getShardUuid() - { - return shardUuid; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/OrcTestingUtil.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/OrcTestingUtil.java deleted file mode 100644 index 8a209cadd2d6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/OrcTestingUtil.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.primitives.UnsignedBytes; -import io.airlift.units.DataSize; -import io.trino.orc.FileOrcDataSource; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcPredicate; -import io.trino.orc.OrcReader; -import io.trino.orc.OrcReaderOptions; -import io.trino.orc.OrcRecordReader; -import io.trino.spi.type.Type; -import org.joda.time.DateTimeZone; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.List; - -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static io.trino.orc.OrcReader.MAX_BATCH_SIZE; -import static org.assertj.core.api.Assertions.assertThat; - -final class OrcTestingUtil -{ - private OrcTestingUtil() {} - - private static final OrcReaderOptions READER_OPTIONS = new OrcReaderOptions() - .withMaxReadBlockSize(DataSize.of(1, MEGABYTE)) - .withMaxMergeDistance(DataSize.of(1, MEGABYTE)) - .withMaxBufferSize(DataSize.of(1, MEGABYTE)) - .withStreamBufferSize(DataSize.of(1, MEGABYTE)) - .withTinyStripeThreshold(DataSize.of(1, MEGABYTE)); - - public static OrcDataSource fileOrcDataSource(File file) - throws FileNotFoundException - { - return new FileOrcDataSource(file, READER_OPTIONS); - } - - public static OrcRecordReader createReader(OrcDataSource dataSource, List columnIds, List types) - throws IOException - { - OrcReader orcReader = OrcReader.createOrcReader(dataSource, READER_OPTIONS) - .orElseThrow(() -> new RuntimeException("File is empty")); - - List columnNames = orcReader.getColumnNames(); - assertThat(columnNames.size()).isEqualTo(columnIds.size()); - - return orcReader.createRecordReader( - orcReader.getRootColumn().getNestedColumns(), - types, - OrcPredicate.TRUE, - DateTimeZone.UTC, - newSimpleAggregatedMemoryContext(), - MAX_BATCH_SIZE, - RuntimeException::new); - } - - public static byte[] octets(int... values) - { - byte[] bytes = new byte[values.length]; - for (int i = 0; i < bytes.length; i++) { - bytes[i] = UnsignedBytes.checkedCast(values[i]); - } - return bytes; - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancer.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancer.java deleted file mode 100644 index ab1bc789e346..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancer.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.HashMultiset; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Multiset; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.client.NodeVersion; -import io.trino.metadata.InternalNode; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.metadata.BucketNode; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.storage.BucketBalancer.BucketAssignment; -import io.trino.plugin.raptor.legacy.storage.BucketBalancer.ClusterState; -import io.trino.spi.Node; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.net.URI; -import java.util.List; -import java.util.OptionalLong; -import java.util.stream.Collectors; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertLessThanOrEqual; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.Distribution.serializeColumnTypes; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.spi.type.BigintType.BIGINT; -import static java.util.concurrent.TimeUnit.DAYS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestBucketBalancer -{ - private static final List AVAILABLE_WORKERS = ImmutableList.of("node1", "node2", "node3", "node4", "node5"); - - private Jdbi dbi; - private Handle dummyHandle; - private ShardManager shardManager; - private TestingNodeManager nodeManager; - private MetadataDao metadataDao; - private BucketBalancer balancer; - - @BeforeEach - public void setup() - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - - metadataDao = dbi.onDemand(MetadataDao.class); - nodeManager = new TestingNodeManager(AVAILABLE_WORKERS.stream() - .map(TestBucketBalancer::createTestingNode) - .collect(Collectors.toList())); - - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - shardManager = createShardManager(dbi, nodeSupplier); - balancer = new BucketBalancer(nodeSupplier, shardManager, true, new Duration(1, DAYS), true, true, "test"); - } - - @AfterEach - public void teardown() - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - } - - @Test - public void testSingleDistributionUnbalanced() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - createAssignments(distributionId, AVAILABLE_WORKERS, 10, 3, 1, 1, 1); - - assertBalancing(balancer, 6); - } - - @Test - public void testSingleDistributionSlightlyUnbalanced() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - createAssignments(distributionId, AVAILABLE_WORKERS, 4, 4, 3, 3, 2); - - assertBalancing(balancer, 1); - } - - @Test - public void testSingleDistributionBalanced() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - createAssignments(distributionId, AVAILABLE_WORKERS, 4, 3, 3, 3, 3); - - assertBalancing(balancer, 0); - } - - @Test - public void testSingleDistributionUnbalancedWithDeadNode() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - ImmutableList nodes = ImmutableList.builder().addAll(AVAILABLE_WORKERS).add("node6").build(); - createAssignments(distributionId, nodes, 11, 1, 1, 1, 1, 1); - - assertBalancing(balancer, 8); - } - - @Test - public void testSingleDistributionUnbalancedWithNewNode() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - createAssignments(distributionId, AVAILABLE_WORKERS, 12, 1, 1, 1, 1); - nodeManager.addNode(createTestingNode("node6")); - - assertBalancing(balancer, 9); - } - - @Test - public void testMultipleDistributionUnbalanced() - { - long distributionA = createDistribution("distA", 17); - createBucketedTable("testA", distributionA); - createAssignments(distributionA, AVAILABLE_WORKERS, 11, 3, 1, 1, 1); - - long distributionB = createDistribution("distB", 10); - createBucketedTable("testB", distributionB); - createAssignments(distributionB, AVAILABLE_WORKERS, 8, 2, 0, 0, 0); - - long distributionC = createDistribution("distC", 4); - createBucketedTable("testC", distributionC); - createAssignments(distributionC, AVAILABLE_WORKERS, 2, 2, 0, 0, 0); - - assertBalancing(balancer, 15); - } - - @Test - public void testMultipleDistributionUnbalancedWithDiskSpace() - { - long distributionA = createDistribution("distA", 4); - createBucketedTable("testA", distributionA, DataSize.valueOf("4B")); - createAssignments(distributionA, AVAILABLE_WORKERS, 1, 1, 1, 1, 0); - - long distributionB = createDistribution("distB", 4); - createBucketedTable("testB", distributionB, DataSize.valueOf("4B")); - createAssignments(distributionB, AVAILABLE_WORKERS, 1, 1, 1, 0, 1); - - long distributionC = createDistribution("distC", 2); - createBucketedTable("testC", distributionC, DataSize.valueOf("2B")); - createAssignments(distributionC, AVAILABLE_WORKERS, 0, 0, 0, 2, 0); - - assertBalancing(balancer, 1); - - assertThat(balancer.fetchClusterState().getAssignedBytes().values() - .stream() - .distinct() - .count()).isEqualTo(1); - } - - @Test - public void testMultipleDistributionUnbalancedWithDiskSpace2() - { - long distributionA = createDistribution("distA", 4); - createBucketedTable("testA", distributionA, DataSize.valueOf("4B")); - createAssignments(distributionA, AVAILABLE_WORKERS, 1, 1, 1, 1, 0); - - long distributionB = createDistribution("distB", 4); - createBucketedTable("testB", distributionB, DataSize.valueOf("4B")); - createAssignments(distributionB, AVAILABLE_WORKERS, 2, 1, 1, 0, 0); - - assertBalancing(balancer, 1); - } - - @Test - public void testMultipleDistributionUnbalancedWorstCase() - { - // we will end up with only one bucket on node1 - long distributionA = createDistribution("distA", 4); - createBucketedTable("testA", distributionA, DataSize.valueOf("4B")); - createAssignments(distributionA, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - long distributionB = createDistribution("distB", 4); - createBucketedTable("testB", distributionB, DataSize.valueOf("4B")); - createAssignments(distributionB, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - long distributionC = createDistribution("distC", 4); - createBucketedTable("testC", distributionC, DataSize.valueOf("4B")); - createAssignments(distributionC, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - long distributionD = createDistribution("distD", 4); - createBucketedTable("testD", distributionD, DataSize.valueOf("4B")); - createAssignments(distributionD, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - long distributionE = createDistribution("distE", 4); - createBucketedTable("testE", distributionE, DataSize.valueOf("4B")); - createAssignments(distributionE, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - assertBalancing(balancer, 15); - } - - private static void assertBalancing(BucketBalancer balancer, int expectedMoves) - { - int actualMoves = balancer.balance(); - assertThat(actualMoves).isEqualTo(expectedMoves); - - // check that number of buckets per node is within bounds - ClusterState clusterState = balancer.fetchClusterState(); - for (Distribution distribution : clusterState.getDistributionAssignments().keySet()) { - Multiset allocationCounts = HashMultiset.create(); - clusterState.getDistributionAssignments().get(distribution).stream() - .map(BucketAssignment::getNodeIdentifier) - .forEach(allocationCounts::add); - - double bucketsPerNode = (1.0 * allocationCounts.size()) / clusterState.getActiveNodes().size(); - for (String node : allocationCounts) { - assertGreaterThanOrEqual(allocationCounts.count(node), (int) Math.floor(bucketsPerNode), node + " has fewer buckets than expected"); - assertLessThanOrEqual(allocationCounts.count(node), (int) Math.ceil(bucketsPerNode), node + " has more buckets than expected"); - } - } - - // check stability - assertThat(balancer.balance()).isEqualTo(0); - } - - private long createDistribution(String distributionName, int bucketCount) - { - MetadataDao dao = dbi.onDemand(MetadataDao.class); - long distributionId = dao.insertDistribution(distributionName, serializeColumnTypes(ImmutableList.of(BIGINT)), bucketCount); - shardManager.createBuckets(distributionId, bucketCount); - return distributionId; - } - - private long createBucketedTable(String tableName, long distributionId) - { - return createBucketedTable(tableName, distributionId, DataSize.valueOf("0B")); - } - - private long createBucketedTable(String tableName, long distributionId, DataSize compressedSize) - { - MetadataDao dao = dbi.onDemand(MetadataDao.class); - long tableId = dao.insertTable("test", tableName, false, false, distributionId, 0); - List columnsA = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columnsA, false, OptionalLong.empty()); - - metadataDao.updateTableStats(tableId, 1024, 1024 * 1024 * 1024, compressedSize.toBytes(), compressedSize.toBytes() * 2); - return tableId; - } - - private List createAssignments(long distributionId, List nodes, int... buckets) - { - checkArgument(nodes.size() == buckets.length); - ImmutableList.Builder assignments = ImmutableList.builder(); - int bucketNumber = 0; - for (int i = 0; i < buckets.length; i++) { - for (int j = 0; j < buckets[i]; j++) { - shardManager.updateBucketAssignment(distributionId, bucketNumber, nodes.get(i)); - assignments.add(bucketNode(bucketNumber, nodes.get(i))); - - bucketNumber++; - } - } - return assignments.build(); - } - - private static BucketNode bucketNode(int bucketNumber, String nodeIdentifier) - { - return new BucketNode(bucketNumber, nodeIdentifier); - } - - private static Node createTestingNode(String nodeIdentifier) - { - return new InternalNode(nodeIdentifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancerConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancerConfig.java deleted file mode 100644 index 6b91e65febf6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancerConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.HOURS; - -public class TestBucketBalancerConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(BucketBalancerConfig.class) - .setBalancerEnabled(true) - .setBalancerInterval(new Duration(6, HOURS))); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("storage.balancer-enabled", "false") - .put("storage.balancer-interval", "5h") - .buildOrThrow(); - - BucketBalancerConfig expected = new BucketBalancerConfig() - .setBalancerEnabled(false) - .setBalancerInterval(new Duration(5, HOURS)); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestFileStorageService.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestFileStorageService.java deleted file mode 100644 index 8fddeefa3818..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestFileStorageService.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableSet; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.storage.FileStorageService.getFileSystemPath; -import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestFileStorageService -{ - private Path temporary; - private FileStorageService store; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - store = new FileStorageService(temporary.toFile()); - store.start(); - } - - @AfterEach - public void tearDown() - throws Exception - { - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testGetFileSystemPath() - { - UUID uuid = UUID.fromString("701e1a79-74f7-4f56-b438-b41e8e7d019d"); - File expected = new File("/test", format("70/1e/%s.orc", uuid)); - assertThat(getFileSystemPath(new File("/test"), uuid)).isEqualTo(expected); - } - - @Test - public void testFilePaths() - { - UUID uuid = UUID.fromString("701e1a79-74f7-4f56-b438-b41e8e7d019d"); - File staging = temporary.resolve("staging").resolve(format("%s.orc", uuid)).toFile(); - File storage = temporary.resolve("storage").resolve("70").resolve("1e").resolve(format("%s.orc", uuid)).toFile(); - File quarantine = temporary.resolve("quarantine").resolve(format("%s.orc", uuid)).toFile(); - assertThat(store.getStagingFile(uuid)).isEqualTo(staging); - assertThat(store.getStorageFile(uuid)).isEqualTo(storage); - assertThat(store.getQuarantineFile(uuid)).isEqualTo(quarantine); - } - - @Test - public void testStop() - throws Exception - { - File staging = temporary.resolve("staging").toFile(); - File storage = temporary.resolve("storage").toFile(); - File quarantine = temporary.resolve("quarantine").toFile(); - - assertThat(staging).isDirectory(); - assertThat(storage).isDirectory(); - assertThat(quarantine).isDirectory(); - - File file = store.getStagingFile(randomUUID()); - store.createParents(file); - assertThat(file).doesNotExist(); - assertThat(file.createNewFile()).isTrue(); - assertThat(file).isFile(); - - store.stop(); - - assertThat(file).doesNotExist(); - assertThat(staging).doesNotExist(); - assertThat(storage).isDirectory(); - assertThat(quarantine).isDirectory(); - } - - @Test - public void testGetStorageShards() - throws Exception - { - Set shards = ImmutableSet.builder() - .add(UUID.fromString("9e7abb51-56b5-4180-9164-ad08ddfe7c63")) - .add(UUID.fromString("bbfc3895-1c3d-4bf4-bca4-7b1198b1759e")) - .build(); - - for (UUID shard : shards) { - File file = store.getStorageFile(shard); - store.createParents(file); - assertThat(file.createNewFile()).isTrue(); - } - - File storage = temporary.resolve("storage").toFile(); - assertThat(new File(storage, "abc").mkdir()).isTrue(); - assertThat(new File(storage, "ab/cd").mkdirs()).isTrue(); - assertThat(new File(storage, format("ab/cd/%s.junk", randomUUID())).createNewFile()).isTrue(); - assertThat(new File(storage, "ab/cd/junk.orc").createNewFile()).isTrue(); - - assertThat(store.getStorageShards()).isEqualTo(shards); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestMissingShardComparator.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestMissingShardComparator.java deleted file mode 100644 index 7c779032c5e9..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestMissingShardComparator.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import org.junit.jupiter.api.Test; - -import static io.trino.plugin.raptor.legacy.storage.ShardRecoveryManager.MissingShardComparator; -import static io.trino.plugin.raptor.legacy.storage.ShardRecoveryManager.MissingShardRunnable; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestMissingShardComparator -{ - @Test - public void testOrdering() - { - MissingShardComparator comparator = new MissingShardComparator(); - assertThat(comparator.compare(new DummyMissingShardRunnable(false), new DummyMissingShardRunnable(false))).isEqualTo(0); - assertThat(comparator.compare(new DummyMissingShardRunnable(false), new DummyMissingShardRunnable(true))).isEqualTo(1); - assertThat(comparator.compare(new DummyMissingShardRunnable(true), new DummyMissingShardRunnable(false))).isEqualTo(-1); - assertThat(comparator.compare(new DummyMissingShardRunnable(true), new DummyMissingShardRunnable(true))).isEqualTo(0); - } - - private static class DummyMissingShardRunnable - implements MissingShardRunnable - { - private final boolean active; - - DummyMissingShardRunnable(boolean active) - { - this.active = active; - } - - @Override - public boolean isActive() - { - return active; - } - - @Override - public void run() - { - throw new UnsupportedOperationException(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcFileRewriter.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcFileRewriter.java deleted file mode 100644 index 08173185ca61..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcFileRewriter.java +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcRecordReader; -import io.trino.plugin.raptor.legacy.storage.OrcFileRewriter.OrcFileInfo; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.SqlMap; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeSignatureParameter; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.math.BigDecimal; -import java.nio.file.Path; -import java.util.BitSet; -import java.util.List; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.RowPagesBuilder.rowPagesBuilder; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.createReader; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.fileOrcDataSource; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.StructuralTestUtil.arrayBlockOf; -import static io.trino.testing.StructuralTestUtil.arrayBlocksEqual; -import static io.trino.testing.StructuralTestUtil.sqlMapEqual; -import static io.trino.testing.StructuralTestUtil.sqlMapOf; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static java.nio.file.Files.createTempDirectory; -import static java.nio.file.Files.readAllBytes; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_CLASS) -@Execution(SAME_THREAD) -public class TestOrcFileRewriter -{ - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - private final Path temporary; - - public TestOrcFileRewriter() - throws IOException - { - temporary = createTempDirectory(null); - } - - @AfterAll - public void tearDown() - throws Exception - { - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testRewrite() - throws Exception - { - ArrayType arrayType = new ArrayType(BIGINT); - ArrayType arrayOfArrayType = new ArrayType(arrayType); - Type mapType = TESTING_TYPE_MANAGER.getParameterizedType(StandardTypes.MAP, ImmutableList.of( - TypeSignatureParameter.typeParameter(createVarcharType(5).getTypeSignature()), - TypeSignatureParameter.typeParameter(BOOLEAN.getTypeSignature()))); - List columnIds = ImmutableList.of(3L, 7L, 9L, 10L, 11L, 12L); - DecimalType decimalType = DecimalType.createDecimalType(4, 4); - - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), arrayType, mapType, arrayOfArrayType, decimalType); - - File file = temporary.resolve(randomUUID().toString()).toFile(); - try (OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello", arrayBlockOf(BIGINT, 1, 2), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), new BigDecimal("2.3")) - .row(777L, "sky", arrayBlockOf(BIGINT, 3, 4), sqlMapOf(createVarcharType(5), BOOLEAN, "k2", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)), new BigDecimal("2.3")) - .row(456L, "bye", arrayBlockOf(BIGINT, 5, 6), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7)), new BigDecimal("2.3")) - .row(888L, "world", arrayBlockOf(BIGINT, 7, 8), sqlMapOf(createVarcharType(5), BOOLEAN, "k4", true), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 8), null), new BigDecimal("2.3")) - .row(999L, "done", arrayBlockOf(BIGINT, 9, 10), sqlMapOf(createVarcharType(5), BOOLEAN, "k5", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 9, 10)), new BigDecimal("2.3")) - .build(); - writer.appendPages(pages); - } - - try (OrcDataSource dataSource = fileOrcDataSource(file)) { - OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); - - assertThat(reader.getReaderRowCount()).isEqualTo(5); - assertThat(reader.getFileRowCount()).isEqualTo(5); - assertThat(reader.getSplitLength()).isEqualTo(file.length()); - - Page page = reader.nextPage(); - assertThat(page.getPositionCount()).isEqualTo(5); - - Block column0 = page.getBlock(0); - assertThat(column0.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column0.isNull(i)).isEqualTo(false); - } - assertThat(BIGINT.getLong(column0, 0)).isEqualTo(123L); - assertThat(BIGINT.getLong(column0, 1)).isEqualTo(777L); - assertThat(BIGINT.getLong(column0, 2)).isEqualTo(456L); - assertThat(BIGINT.getLong(column0, 3)).isEqualTo(888L); - assertThat(BIGINT.getLong(column0, 4)).isEqualTo(999L); - - Block column1 = page.getBlock(1); - assertThat(column1.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column1.isNull(i)).isEqualTo(false); - } - assertThat(createVarcharType(20).getSlice(column1, 0)).isEqualTo(utf8Slice("hello")); - assertThat(createVarcharType(20).getSlice(column1, 1)).isEqualTo(utf8Slice("sky")); - assertThat(createVarcharType(20).getSlice(column1, 2)).isEqualTo(utf8Slice("bye")); - assertThat(createVarcharType(20).getSlice(column1, 3)).isEqualTo(utf8Slice("world")); - assertThat(createVarcharType(20).getSlice(column1, 4)).isEqualTo(utf8Slice("done")); - - Block column2 = page.getBlock(2); - assertThat(column2.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column2.isNull(i)).isEqualTo(false); - } - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 0), arrayBlockOf(BIGINT, 1, 2))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 1), arrayBlockOf(BIGINT, 3, 4))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 2), arrayBlockOf(BIGINT, 5, 6))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 3), arrayBlockOf(BIGINT, 7, 8))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 4), arrayBlockOf(BIGINT, 9, 10))).isTrue(); - - Block column3 = page.getBlock(3); - assertThat(column3.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column3.isNull(i)).isEqualTo(false); - } - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 0), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 1), sqlMapOf(createVarcharType(5), BOOLEAN, "k2", false))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 2), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", true))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 3), sqlMapOf(createVarcharType(5), BOOLEAN, "k4", true))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 4), sqlMapOf(createVarcharType(5), BOOLEAN, "k5", true))).isTrue(); - - Block column4 = page.getBlock(4); - assertThat(column4.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column4.isNull(i)).isEqualTo(false); - } - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 1), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 3), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 8), null))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 4), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 9, 10)))).isTrue(); - - assertThat(reader.nextPage()).isNull(); - - OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes()); - assertThat(orcFileMetadata).isEqualTo(new OrcFileMetadata(ImmutableMap.builder() - .put(3L, BIGINT.getTypeId()) - .put(7L, createVarcharType(20).getTypeId()) - .put(9L, arrayType.getTypeId()) - .put(10L, mapType.getTypeId()) - .put(11L, arrayOfArrayType.getTypeId()) - .put(12L, decimalType.getTypeId()) - .buildOrThrow())); - } - - BitSet rowsToDelete = new BitSet(5); - rowsToDelete.set(1); - rowsToDelete.set(3); - rowsToDelete.set(4); - - File newFile = temporary.resolve(randomUUID().toString()).toFile(); - OrcFileInfo info = OrcFileRewriter.rewrite(TESTING_TYPE_MANAGER, file, newFile, rowsToDelete); - assertThat(info.getRowCount()).isEqualTo(2); - assertThat(info.getUncompressedSize()).isEqualTo(182); - - try (OrcDataSource dataSource = fileOrcDataSource(newFile)) { - OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); - - assertThat(reader.getReaderRowCount()).isEqualTo(2); - assertThat(reader.getFileRowCount()).isEqualTo(2); - assertThat(reader.getSplitLength()).isEqualTo(newFile.length()); - - Page page = reader.nextPage(); - assertThat(page.getPositionCount()).isEqualTo(2); - - Block column0 = page.getBlock(0); - assertThat(column0.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column0.isNull(i)).isEqualTo(false); - } - assertThat(BIGINT.getLong(column0, 0)).isEqualTo(123L); - assertThat(BIGINT.getLong(column0, 1)).isEqualTo(456L); - - Block column1 = page.getBlock(1); - assertThat(column1.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column1.isNull(i)).isEqualTo(false); - } - assertThat(createVarcharType(20).getSlice(column1, 0)).isEqualTo(utf8Slice("hello")); - assertThat(createVarcharType(20).getSlice(column1, 1)).isEqualTo(utf8Slice("bye")); - - Block column2 = page.getBlock(2); - assertThat(column2.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column2.isNull(i)).isEqualTo(false); - } - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 0), arrayBlockOf(BIGINT, 1, 2))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 1), arrayBlockOf(BIGINT, 5, 6))).isTrue(); - - Block column3 = page.getBlock(3); - assertThat(column3.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column3.isNull(i)).isEqualTo(false); - } - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 0), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 1), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", true))).isTrue(); - - Block column4 = page.getBlock(4); - assertThat(column4.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column4.isNull(i)).isEqualTo(false); - } - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 1), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7)))).isTrue(); - - assertThat(reader.nextPage()).isEqualTo(null); - - OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes()); - assertThat(orcFileMetadata).isEqualTo(new OrcFileMetadata(ImmutableMap.builder() - .put(3L, BIGINT.getTypeId()) - .put(7L, createVarcharType(20).getTypeId()) - .put(9L, arrayType.getTypeId()) - .put(10L, mapType.getTypeId()) - .put(11L, arrayOfArrayType.getTypeId()) - .put(12L, decimalType.getTypeId()) - .buildOrThrow())); - } - } - - @Test - public void testRewriteAllRowsDeleted() - throws Exception - { - List columnIds = ImmutableList.of(3L); - List columnTypes = ImmutableList.of(BIGINT); - - File file = temporary.resolve(randomUUID().toString()).toFile(); - try (OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); - } - - BitSet rowsToDelete = new BitSet(); - rowsToDelete.set(0); - rowsToDelete.set(1); - - File newFile = temporary.resolve(randomUUID().toString()).toFile(); - OrcFileInfo info = OrcFileRewriter.rewrite(TESTING_TYPE_MANAGER, file, newFile, rowsToDelete); - assertThat(info.getRowCount()).isEqualTo(0); - assertThat(info.getUncompressedSize()).isEqualTo(0); - - assertThat(newFile).doesNotExist(); - } - - @Test - public void testRewriteNoRowsDeleted() - throws Exception - { - List columnIds = ImmutableList.of(3L); - List columnTypes = ImmutableList.of(BIGINT); - - File file = temporary.resolve(randomUUID().toString()).toFile(); - try (OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); - } - - BitSet rowsToDelete = new BitSet(); - - File newFile = temporary.resolve(randomUUID().toString()).toFile(); - OrcFileInfo info = OrcFileRewriter.rewrite(TESTING_TYPE_MANAGER, file, newFile, rowsToDelete); - assertThat(info.getRowCount()).isEqualTo(2); - assertThat(info.getUncompressedSize()).isEqualTo(18); - - assertThat(readAllBytes(newFile.toPath())).isEqualTo(readAllBytes(file.toPath())); - } - - @Test - public void testUncompressedSize() - throws Exception - { - List columnIds = ImmutableList.of(1L, 2L, 3L, 4L, 5L); - List columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY); - - File file = temporary.resolve(randomUUID().toString()).toFile(); - try (OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - List pages = rowPagesBuilder(columnTypes) - .row(true, 123L, 98.7, "hello", utf8Slice("abc")) - .row(false, 456L, 65.4, "world", utf8Slice("xyz")) - .row(null, null, null, null, null) - .build(); - writer.appendPages(pages); - } - - File newFile = temporary.resolve(randomUUID().toString()).toFile(); - OrcFileInfo info = OrcFileRewriter.rewrite(TESTING_TYPE_MANAGER, file, newFile, new BitSet()); - assertThat(info.getRowCount()).isEqualTo(3); - assertThat(info.getUncompressedSize()).isEqualTo(106); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java deleted file mode 100644 index e1fcc36158f6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java +++ /dev/null @@ -1,692 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import io.airlift.slice.Slice; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcReaderOptions; -import io.trino.orc.OrcRecordReader; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.backup.BackupManager; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.backup.FileBackupStore; -import io.trino.plugin.raptor.legacy.metadata.ColumnStats; -import io.trino.plugin.raptor.legacy.metadata.ShardDelta; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; -import io.trino.plugin.raptor.legacy.storage.InMemoryShardRecorder.RecordedShard; -import io.trino.spi.NodeManager; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.predicate.NullableValue; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.SqlDate; -import io.trino.spi.type.SqlTimestamp; -import io.trino.spi.type.SqlVarbinary; -import io.trino.spi.type.Type; -import io.trino.testing.MaterializedResult; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.joda.time.DateTime; -import org.joda.time.Days; -import org.joda.time.chrono.ISOChronology; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.BitSet; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.slice.Slices.wrappedBuffer; -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static io.trino.RowPagesBuilder.rowPagesBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.createReader; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.octets; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.DateTimeTestingUtils.sqlTimestampOf; -import static io.trino.testing.MaterializedResult.materializeSourceDataStream; -import static io.trino.testing.MaterializedResult.resultBuilder; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Fail.fail; -import static org.joda.time.DateTimeZone.UTC; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestRaptorStorageManager -{ - private static final ISOChronology UTC_CHRONOLOGY = ISOChronology.getInstanceUTC(); - private static final DateTime EPOCH = new DateTime(0, UTC_CHRONOLOGY); - private static final String CURRENT_NODE = "node"; - private static final String CONNECTOR_ID = "test"; - private static final long TRANSACTION_ID = 123; - private static final int DELETION_THREADS = 2; - private static final Duration SHARD_RECOVERY_TIMEOUT = new Duration(30, TimeUnit.SECONDS); - private static final int MAX_SHARD_ROWS = 100; - private static final DataSize MAX_FILE_SIZE = DataSize.of(1, MEGABYTE); - private static final Duration MISSING_SHARD_DISCOVERY = new Duration(5, TimeUnit.MINUTES); - private static final OrcReaderOptions READER_OPTIONS = new OrcReaderOptions() - .withMaxMergeDistance(DataSize.of(1, MEGABYTE)) - .withMaxBufferSize(DataSize.of(1, MEGABYTE)) - .withStreamBufferSize(DataSize.of(1, MEGABYTE)) - .withTinyStripeThreshold(DataSize.of(1, MEGABYTE)); - - private final NodeManager nodeManager = new TestingNodeManager(); - private Handle dummyHandle; - private Path temporary; - private StorageService storageService; - private ShardRecoveryManager recoveryManager; - private FileBackupStore fileBackupStore; - private Optional backupStore; - private InMemoryShardRecorder shardRecorder; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - File directory = temporary.resolve("data").toFile(); - storageService = new FileStorageService(directory); - storageService.start(); - - File backupDirectory = temporary.resolve("backup").toFile(); - fileBackupStore = new FileBackupStore(backupDirectory); - fileBackupStore.start(); - backupStore = Optional.of(fileBackupStore); - - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - - ShardManager shardManager = createShardManager(dbi); - Duration discoveryInterval = new Duration(5, TimeUnit.MINUTES); - recoveryManager = new ShardRecoveryManager(storageService, backupStore, nodeManager, shardManager, discoveryInterval, 10); - - shardRecorder = new InMemoryShardRecorder(); - } - - @AfterEach - public void tearDown() - throws Exception - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testWriter() - throws Exception - { - RaptorStorageManager manager = createRaptorStorageManager(); - - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); - - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - sink.appendPages(pages); - - // shard is not recorded until flush - assertThat(shardRecorder.getShards().size()).isEqualTo(0); - - sink.flush(); - - // shard is recorded after flush - List recordedShards = shardRecorder.getShards(); - assertThat(recordedShards.size()).isEqualTo(1); - - List shards = getFutureValue(sink.commit()); - - assertThat(shards.size()).isEqualTo(1); - ShardInfo shardInfo = Iterables.getOnlyElement(shards); - - UUID shardUuid = shardInfo.getShardUuid(); - File file = storageService.getStorageFile(shardUuid); - File backupFile = fileBackupStore.getBackupFile(shardUuid); - - assertThat(recordedShards.get(0).getTransactionId()).isEqualTo(TRANSACTION_ID); - assertThat(recordedShards.get(0).getShardUuid()).isEqualTo(shardUuid); - - assertThat(shardInfo.getRowCount()).isEqualTo(2); - assertThat(shardInfo.getCompressedSize()).isEqualTo(file.length()); - assertThat(shardInfo.getXxhash64()).isEqualTo(xxhash64(file)); - - // verify primary and backup shard exist - assertThat(file) - .describedAs("primary shard") - .exists(); - assertThat(backupFile) - .describedAs("backup shard") - .exists(); - - assertFileEquals(file, backupFile); - - // remove primary shard to force recovery from backup - assertThat(file.delete()).isTrue(); - assertThat(file.getParentFile().delete()).isTrue(); - assertThat(file).doesNotExist(); - - recoveryManager.restoreFromBackup(shardUuid, shardInfo.getCompressedSize(), OptionalLong.of(shardInfo.getXxhash64())); - - try (OrcDataSource dataSource = manager.openShard(shardUuid, READER_OPTIONS)) { - OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); - - Page page = reader.nextPage(); - assertThat(page.getPositionCount()).isEqualTo(2); - - Block column0 = page.getBlock(0); - assertThat(column0.isNull(0)).isEqualTo(false); - assertThat(column0.isNull(1)).isEqualTo(false); - assertThat(BIGINT.getLong(column0, 0)).isEqualTo(123L); - assertThat(BIGINT.getLong(column0, 1)).isEqualTo(456L); - - Block column1 = page.getBlock(1); - assertThat(createVarcharType(10).getSlice(column1, 0)).isEqualTo(utf8Slice("hello")); - assertThat(createVarcharType(10).getSlice(column1, 1)).isEqualTo(utf8Slice("bye")); - - assertThat(reader.nextPage()).isNull(); - } - } - - @Test - public void testReader() - throws Exception - { - RaptorStorageManager manager = createRaptorStorageManager(); - - List columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE); - - byte[] bytes1 = octets(0x00, 0xFE, 0xFF); - byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80); - - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - - Object[][] doubles = { - {881L, "-inf", null, null, null, Double.NEGATIVE_INFINITY}, - {882L, "+inf", null, null, null, Double.POSITIVE_INFINITY}, - {883L, "nan", null, null, null, Double.NaN}, - {884L, "min", null, null, null, Double.MIN_VALUE}, - {885L, "max", null, null, null, Double.MAX_VALUE}, - {886L, "pzero", null, null, null, 0.0}, - {887L, "nzero", null, null, null, -0.0}, - }; - - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello", wrappedBuffer(bytes1), sqlDate(2001, 8, 22).getDays(), true, 123.45) - .row(null, null, null, null, null, null) - .row(456L, "bye", wrappedBuffer(bytes3), sqlDate(2005, 4, 22).getDays(), false, 987.65) - .rows(doubles) - .build(); - - sink.appendPages(pages); - List shards = getFutureValue(sink.commit()); - - assertThat(shards.size()).isEqualTo(1); - UUID uuid = Iterables.getOnlyElement(shards).getShardUuid(); - - MaterializedResult expected = resultBuilder(SESSION, columnTypes) - .row(123L, "hello", sqlBinary(bytes1), sqlDate(2001, 8, 22), true, 123.45) - .row(null, null, null, null, null, null) - .row(456L, "bye", sqlBinary(bytes3), sqlDate(2005, 4, 22), false, 987.65) - .rows(doubles) - .build(); - - // no tuple domain (all) - TupleDomain tupleDomain = TupleDomain.all(); - - try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) { - MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); - assertThat(result.getRowCount()).isEqualTo(expected.getRowCount()); - assertThat(result).containsExactlyElementsOf(expected); - } - - // tuple domain within the column range - tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(new RaptorColumnHandle("c1", 2, BIGINT), NullableValue.of(BIGINT, 124L))); - - try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) { - MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); - assertThat(result.getRowCount()).isEqualTo(expected.getRowCount()); - } - - // tuple domain outside the column range - tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(new RaptorColumnHandle("c1", 2, BIGINT), NullableValue.of(BIGINT, 122L))); - - try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) { - MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); - assertThat(result.getRowCount()).isEqualTo(0); - } - } - - @Test - public void testRewriter() - throws Exception - { - RaptorStorageManager manager = createRaptorStorageManager(); - - long transactionId = TRANSACTION_ID; - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); - - // create file with 2 rows - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - sink.appendPages(pages); - List shards = getFutureValue(sink.commit()); - - assertThat(shardRecorder.getShards().size()).isEqualTo(1); - - // delete one row - BitSet rowsToDelete = new BitSet(); - rowsToDelete.set(0); - Collection fragments = manager.rewriteShard(transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete); - - Slice shardDelta = Iterables.getOnlyElement(fragments); - ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes()); - ShardInfo shardInfo = Iterables.getOnlyElement(shardDeltas.getNewShards()); - - // check that output file has one row - assertThat(shardInfo.getRowCount()).isEqualTo(1); - - // check that storage file is same as backup file - File storageFile = storageService.getStorageFile(shardInfo.getShardUuid()); - File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid()); - assertFileEquals(storageFile, backupFile); - - // verify recorded shard - List recordedShards = shardRecorder.getShards(); - assertThat(recordedShards.size()).isEqualTo(2); - assertThat(recordedShards.get(1).getTransactionId()).isEqualTo(TRANSACTION_ID); - assertThat(recordedShards.get(1).getShardUuid()).isEqualTo(shardInfo.getShardUuid()); - } - - @Test - public void testWriterRollback() - { - // verify staging directory is empty - File staging = temporary.resolve("data").resolve("staging").toFile(); - assertThat(staging).isDirectory(); - assertThat(staging.list()).isEqualTo(new String[] {}); - - // create a shard in staging - RaptorStorageManager manager = createRaptorStorageManager(); - - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); - - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - sink.appendPages(pages); - - sink.flush(); - - // verify shard exists in staging - String[] files = staging.list(); - assertThat(files).isNotNull(); - String stagingFile = Arrays.stream(files) - .filter(file -> file.endsWith(".orc")) - .findFirst() - .orElseThrow(() -> new AssertionError("file not found in staging")); - - // rollback should cleanup staging files - sink.rollback(); - - files = staging.list(); - assertThat(files).isNotNull(); - assertThat(Arrays.stream(files).noneMatch(stagingFile::equals)).isTrue(); - } - - @Test - public void testShardStatsBigint() - { - List stats = columnStats(types(BIGINT), - row(2L), - row(-3L), - row(5L)); - assertColumnStats(stats, 1, -3L, 5L); - } - - @Test - public void testShardStatsDouble() - { - List stats = columnStats(types(DOUBLE), - row(2.5), - row(-4.1), - row(6.6)); - assertColumnStats(stats, 1, -4.1, 6.6); - } - - @Test - public void testShardStatsBigintDouble() - { - List stats = columnStats(types(BIGINT, DOUBLE), - row(-3L, 6.6), - row(5L, -4.1)); - assertColumnStats(stats, 1, -3L, 5L); - assertColumnStats(stats, 2, -4.1, 6.6); - } - - @Test - public void testShardStatsDoubleMinMax() - { - List stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.MIN_VALUE), - row(4.5)); - assertColumnStats(stats, 1, Double.MIN_VALUE, 4.5); - - stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.MAX_VALUE), - row(4.5)); - assertColumnStats(stats, 1, 3.2, Double.MAX_VALUE); - } - - @Test - public void testShardStatsDoubleNotFinite() - { - List stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.NEGATIVE_INFINITY), - row(4.5)); - assertColumnStats(stats, 1, null, 4.5); - - stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.POSITIVE_INFINITY), - row(4.5)); - assertColumnStats(stats, 1, 3.2, null); - - stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.NaN), - row(4.5)); - assertColumnStats(stats, 1, 3.2, 4.5); - } - - @Test - public void testShardStatsVarchar() - { - List stats = columnStats( - types(createVarcharType(10)), - row(utf8Slice("hello")), - row(utf8Slice("bye")), - row(utf8Slice("foo"))); - assertColumnStats(stats, 1, "bye", "hello"); - } - - @Test - public void testShardStatsBigintVarbinary() - { - List stats = columnStats(types(BIGINT, VARBINARY), - row(5L, wrappedBuffer(octets(0x00))), - row(3L, wrappedBuffer(octets(0x01)))); - assertColumnStats(stats, 1, 3L, 5L); - assertNoColumnStats(stats, 2); - } - - @Test - public void testShardStatsDateTimestamp() - { - long minDate = sqlDate(2001, 8, 22).getDays(); - long maxDate = sqlDate(2005, 4, 22).getDays(); - long maxTimestamp = sqlTimestamp(2002, 4, 13, 6, 7, 8).getMillis(); - long minTimestamp = sqlTimestamp(2001, 3, 15, 9, 10, 11).getMillis(); - - List stats = columnStats(types(DATE, TIMESTAMP_MILLIS), - row(minDate, maxTimestamp), - row(maxDate, minTimestamp)); - assertColumnStats(stats, 1, minDate, maxDate); - assertColumnStats(stats, 2, minTimestamp, maxTimestamp); - } - - @Test - public void testMaxShardRows() - { - RaptorStorageManager manager = createRaptorStorageManager(2, DataSize.of(2, MEGABYTE)); - - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); - - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - sink.appendPages(pages); - assertThat(sink.isFull()).isTrue(); - } - - @Test - public void testMaxFileSize() - { - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(5)); - - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - - // Set maxFileSize to 1 byte, so adding any page makes the StoragePageSink full - RaptorStorageManager manager = createRaptorStorageManager(20, DataSize.ofBytes(1)); - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - sink.appendPages(pages); - assertThat(sink.isFull()).isTrue(); - } - - private static ConnectorPageSource getPageSource( - RaptorStorageManager manager, - List columnIds, - List columnTypes, - UUID uuid, - TupleDomain tupleDomain) - { - return manager.getPageSource(uuid, OptionalInt.empty(), columnIds, columnTypes, tupleDomain, READER_OPTIONS); - } - - private static StoragePageSink createStoragePageSink(StorageManager manager, List columnIds, List columnTypes) - { - long transactionId = TRANSACTION_ID; - return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); - } - - private RaptorStorageManager createRaptorStorageManager() - { - return createRaptorStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE); - } - - private RaptorStorageManager createRaptorStorageManager(int maxShardRows, DataSize maxFileSize) - { - return createRaptorStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize); - } - - public static RaptorStorageManager createRaptorStorageManager(Jdbi dbi, File temporary) - { - return createRaptorStorageManager(dbi, temporary, MAX_SHARD_ROWS); - } - - public static RaptorStorageManager createRaptorStorageManager(Jdbi dbi, File temporary, int maxShardRows) - { - File directory = new File(temporary, "data"); - StorageService storageService = new FileStorageService(directory); - storageService.start(); - - File backupDirectory = new File(temporary, "backup"); - FileBackupStore fileBackupStore = new FileBackupStore(backupDirectory); - fileBackupStore.start(); - Optional backupStore = Optional.of(fileBackupStore); - - ShardManager shardManager = createShardManager(dbi); - ShardRecoveryManager recoveryManager = new ShardRecoveryManager( - storageService, - backupStore, - new TestingNodeManager(), - shardManager, - MISSING_SHARD_DISCOVERY, - 10); - return createRaptorStorageManager( - storageService, - backupStore, - recoveryManager, - new InMemoryShardRecorder(), - maxShardRows, - MAX_FILE_SIZE); - } - - public static RaptorStorageManager createRaptorStorageManager( - StorageService storageService, - Optional backupStore, - ShardRecoveryManager recoveryManager, - ShardRecorder shardRecorder, - int maxShardRows, - DataSize maxFileSize) - { - return new RaptorStorageManager( - CURRENT_NODE, - storageService, - backupStore, - READER_OPTIONS, - new BackupManager(backupStore, storageService, 1), - recoveryManager, - shardRecorder, - TESTING_TYPE_MANAGER, - CONNECTOR_ID, - DELETION_THREADS, - SHARD_RECOVERY_TIMEOUT, - maxShardRows, - maxFileSize, - DataSize.ofBytes(0)); - } - - private static void assertFileEquals(File actual, File expected) - { - assertThat(actual).hasSameBinaryContentAs(expected); - } - - private static void assertColumnStats(List list, long columnId, Object min, Object max) - { - for (ColumnStats stats : list) { - if (stats.getColumnId() == columnId) { - assertThat(stats.getMin()).isEqualTo(min); - assertThat(stats.getMax()).isEqualTo(max); - return; - } - } - fail(format("no stats for column: %s: %s", columnId, list)); - } - - private static void assertNoColumnStats(List list, long columnId) - { - for (ColumnStats stats : list) { - assertThat(stats.getColumnId()) - .isNotEqualTo(columnId); - } - } - - private static List types(Type... types) - { - return ImmutableList.copyOf(types); - } - - private static Object[] row(Object... values) - { - return values; - } - - private List columnStats(List columnTypes, Object[]... rows) - { - ImmutableList.Builder list = ImmutableList.builder(); - for (long i = 1; i <= columnTypes.size(); i++) { - list.add(i); - } - List columnIds = list.build(); - - RaptorStorageManager manager = createRaptorStorageManager(); - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - sink.appendPages(rowPagesBuilder(columnTypes).rows(rows).build()); - List shards = getFutureValue(sink.commit()); - - assertThat(shards.size()).isEqualTo(1); - return Iterables.getOnlyElement(shards).getColumnStats(); - } - - private static SqlVarbinary sqlBinary(byte[] bytes) - { - return new SqlVarbinary(bytes); - } - - private static SqlDate sqlDate(int year, int month, int day) - { - DateTime date = new DateTime(year, month, day, 0, 0, 0, 0, UTC); - return new SqlDate(Days.daysBetween(EPOCH, date).getDays()); - } - - private static SqlTimestamp sqlTimestamp(int year, int month, int day, int hour, int minute, int second) - { - return sqlTimestampOf(3, year, month, day, hour, minute, second, 0); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardEjector.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardEjector.java deleted file mode 100644 index 295d971eb50c..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardEjector.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.units.Duration; -import io.trino.client.NodeVersion; -import io.trino.metadata.InternalNode; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.predicate.TupleDomain; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.RepeatedTest; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.spi.type.BigintType.BIGINT; -import static java.nio.file.Files.createTempDirectory; -import static java.util.UUID.randomUUID; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardEjector -{ - private Jdbi dbi; - private Handle dummyHandle; - private ShardManager shardManager; - private Path dataDir; - private StorageService storageService; - - @BeforeEach - public void setup() - throws IOException - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - shardManager = createShardManager(dbi); - - dataDir = createTempDirectory(null); - storageService = new FileStorageService(dataDir.toFile()); - storageService.start(); - } - - @AfterEach - public void teardown() - throws Exception - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - if (dataDir != null) { - deleteRecursively(dataDir, ALLOW_INSECURE); - } - } - - @RepeatedTest(20) - public void testEjector() - throws Exception - { - NodeManager nodeManager = createNodeManager("node1", "node2", "node3", "node4", "node5"); - - ShardEjector ejector = new ShardEjector( - nodeManager.getCurrentNode().getNodeIdentifier(), - nodeManager::getWorkerNodes, - shardManager, - storageService, - new Duration(1, HOURS), - Optional.of(new TestingBackupStore()), - "test"); - - List shards = ImmutableList.builder() - .add(shardInfo("node1", 14)) - .add(shardInfo("node1", 13)) - .add(shardInfo("node1", 12)) - .add(shardInfo("node1", 11)) - .add(shardInfo("node1", 10)) - .add(shardInfo("node1", 10)) - .add(shardInfo("node1", 10)) - .add(shardInfo("node1", 10)) - .add(shardInfo("node2", 5)) - .add(shardInfo("node2", 5)) - .add(shardInfo("node3", 10)) - .add(shardInfo("node4", 10)) - .add(shardInfo("node5", 10)) - .add(shardInfo("node6", 200)) - .build(); - - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - for (ShardInfo shard : shards.subList(0, 8)) { - File file = storageService.getStorageFile(shard.getShardUuid()); - storageService.createParents(file); - assertThat(file.createNewFile()).isTrue(); - } - - ejector.process(); - - shardManager.getShardNodes(tableId, TupleDomain.all()); - - Set ejectedShards = shards.subList(0, 4).stream() - .map(ShardInfo::getShardUuid) - .collect(toSet()); - Set keptShards = shards.subList(4, 8).stream() - .map(ShardInfo::getShardUuid) - .collect(toSet()); - - Set remaining = uuids(shardManager.getNodeShards("node1")); - - for (UUID uuid : ejectedShards) { - assertThat(remaining).doesNotContain(uuid); - assertThat(storageService.getStorageFile(uuid)).doesNotExist(); - } - - assertThat(remaining).isEqualTo(keptShards); - for (UUID uuid : keptShards) { - assertThat(storageService.getStorageFile(uuid)).exists(); - } - - Set others = ImmutableSet.builder() - .addAll(uuids(shardManager.getNodeShards("node2"))) - .addAll(uuids(shardManager.getNodeShards("node3"))) - .addAll(uuids(shardManager.getNodeShards("node4"))) - .addAll(uuids(shardManager.getNodeShards("node5"))) - .build(); - - assertThat(others).containsAll(ejectedShards); - } - - private long createTable(String name) - { - return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0); - } - - private static Set uuids(Set metadata) - { - return metadata.stream() - .map(ShardMetadata::getShardUuid) - .collect(toSet()); - } - - private static ShardInfo shardInfo(String node, long size) - { - return new ShardInfo(randomUUID(), OptionalInt.empty(), ImmutableSet.of(node), ImmutableList.of(), 1, size, size * 2, 0); - } - - private static NodeManager createNodeManager(String current, String... others) - { - Node currentNode = createTestingNode(current); - TestingNodeManager nodeManager = new TestingNodeManager(currentNode); - for (String other : others) { - nodeManager.addNode(createTestingNode(other)); - } - return nodeManager; - } - - private static Node createTestingNode(String identifier) - { - return new InternalNode(identifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); - } - - private static class TestingBackupStore - implements BackupStore - { - @Override - public void backupShard(UUID uuid, File source) - { - throw new UnsupportedOperationException(); - } - - @Override - public void restoreShard(UUID uuid, File target) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean deleteShard(UUID uuid) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean shardExists(UUID uuid) - { - return true; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java deleted file mode 100644 index 9199ab498c75..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.io.Files; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.backup.FileBackupStore; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.spi.TrinoException; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.UUID; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; -import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; -import static java.nio.file.Files.createTempDirectory; -import static java.nio.file.Files.createTempFile; -import static java.nio.file.Files.writeString; -import static java.util.concurrent.TimeUnit.MINUTES; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardRecovery -{ - private StorageService storageService; - private ShardRecoveryManager recoveryManager; - private Handle dummyHandle; - private Path temporary; - private FileBackupStore backupStore; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - File directory = temporary.resolve("data").toFile(); - File backupDirectory = temporary.resolve("backup").toFile(); - backupStore = new FileBackupStore(backupDirectory); - backupStore.start(); - storageService = new FileStorageService(directory); - storageService.start(); - - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - ShardManager shardManager = createShardManager(dbi); - recoveryManager = createShardRecoveryManager(storageService, Optional.of(backupStore), shardManager); - } - - @AfterEach - public void tearDown() - throws Exception - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testShardRecovery() - throws Exception - { - UUID shardUuid = UUID.randomUUID(); - File file = storageService.getStorageFile(shardUuid); - File tempFile = createTempFile(temporary, "tmp", null).toFile(); - - writeString(tempFile.toPath(), "test data"); - - backupStore.backupShard(shardUuid, tempFile); - assertThat(backupStore.shardExists(shardUuid)).isTrue(); - File backupFile = backupStore.getBackupFile(shardUuid); - assertThat(backupFile).exists(); - assertThat(backupFile.length()).isEqualTo(tempFile.length()); - - assertThat(file).doesNotExist(); - recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.empty()); - assertThat(file).exists(); - assertThat(file.length()).isEqualTo(tempFile.length()); - } - - @Test - public void testShardRecoveryExistingFileSizeMismatch() - throws Exception - { - UUID shardUuid = UUID.randomUUID(); - - // write data and backup - File tempFile = createTempFile(temporary, "tmp", null).toFile(); - writeString(tempFile.toPath(), "test data"); - - backupStore.backupShard(shardUuid, tempFile); - assertThat(backupStore.shardExists(shardUuid)).isTrue(); - - File backupFile = backupStore.getBackupFile(shardUuid); - assertThat(Files.equal(tempFile, backupFile)).isTrue(); - - // write corrupt storage file with wrong length - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); - - writeString(storageFile.toPath(), "bad data"); - - assertThat(storageFile).exists(); - assertThat(storageFile.length()) - .isNotEqualTo(tempFile.length()); - assertThat(Files.equal(storageFile, tempFile)).isFalse(); - - // restore from backup and verify - recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.empty()); - - assertThat(storageFile).exists(); - assertThat(Files.equal(storageFile, tempFile)).isTrue(); - - // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); - assertThat(quarantined.size()).isEqualTo(1); - assertThat(getOnlyElement(quarantined)).startsWith(shardUuid + ".orc.corrupt"); - } - - @Test - public void testShardRecoveryExistingFileChecksumMismatch() - throws Exception - { - UUID shardUuid = UUID.randomUUID(); - - // write data and backup - File tempFile = createTempFile(temporary, "tmp", null).toFile(); - writeString(tempFile.toPath(), "test data"); - - backupStore.backupShard(shardUuid, tempFile); - assertThat(backupStore.shardExists(shardUuid)).isTrue(); - - File backupFile = backupStore.getBackupFile(shardUuid); - assertThat(Files.equal(tempFile, backupFile)).isTrue(); - - // write corrupt storage file with wrong data - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); - - writeString(storageFile.toPath(), "test xata"); - - assertThat(storageFile).exists(); - assertThat(storageFile.length()).isEqualTo(tempFile.length()); - assertThat(Files.equal(storageFile, tempFile)).isFalse(); - - // restore from backup and verify - recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.of(xxhash64(tempFile))); - - assertThat(storageFile).exists(); - assertThat(Files.equal(storageFile, tempFile)).isTrue(); - - // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); - assertThat(quarantined.size()).isEqualTo(1); - assertThat(getOnlyElement(quarantined)).startsWith(shardUuid + ".orc.corrupt"); - } - - @Test - public void testShardRecoveryBackupChecksumMismatch() - throws Exception - { - UUID shardUuid = UUID.randomUUID(); - - // write storage file - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); - - writeString(storageFile.toPath(), "test data"); - - long size = storageFile.length(); - long xxhash64 = xxhash64(storageFile); - - // backup and verify - backupStore.backupShard(shardUuid, storageFile); - - assertThat(backupStore.shardExists(shardUuid)).isTrue(); - File backupFile = backupStore.getBackupFile(shardUuid); - assertThat(Files.equal(storageFile, backupFile)).isTrue(); - - // corrupt backup file - writeString(backupFile.toPath(), "test xata"); - - assertThat(backupFile).exists(); - assertThat(storageFile.length()).isEqualTo(backupFile.length()); - assertThat(Files.equal(storageFile, backupFile)).isFalse(); - - // delete local file to force restore - assertThat(storageFile.delete()).isTrue(); - assertThat(storageFile).doesNotExist(); - - // restore should fail - assertTrinoExceptionThrownBy(() -> recoveryManager.restoreFromBackup(shardUuid, size, OptionalLong.of(xxhash64))) - .hasErrorCode(RAPTOR_BACKUP_CORRUPTION) - .hasMessage("Backup is corrupt after read: %s", shardUuid); - - // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); - assertThat(quarantined.size()).isEqualTo(1); - assertThat(getOnlyElement(quarantined)).startsWith(shardUuid + ".orc.corrupt"); - } - - @Test - public void testNoBackupException() - { - assertThatThrownBy(() -> { - recoveryManager.restoreFromBackup(UUID.randomUUID(), 0, OptionalLong.empty()); - }) - .isInstanceOf(TrinoException.class) - .hasMessageMatching("No backup file found for shard: .*"); - } - - public static ShardRecoveryManager createShardRecoveryManager( - StorageService storageService, - Optional backupStore, - ShardManager shardManager) - { - return new ShardRecoveryManager( - storageService, - backupStore, - new TestingNodeManager(), - shardManager, - new Duration(5, MINUTES), - 10); - } - - private static List listFiles(File path) - { - String[] files = path.list(); - assertThat(files).isNotNull(); - return ImmutableList.copyOf(files); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardWriter.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardWriter.java deleted file mode 100644 index 7eb3d6e53ed5..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardWriter.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.trino.RowPagesBuilder; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcRecordReader; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.SqlMap; -import io.trino.spi.classloader.ThreadContextClassLoader; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeSignatureParameter; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.slice.Slices.wrappedBuffer; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.createReader; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.fileOrcDataSource; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.octets; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.StructuralTestUtil.arrayBlockOf; -import static io.trino.testing.StructuralTestUtil.arrayBlocksEqual; -import static io.trino.testing.StructuralTestUtil.sqlMapEqual; -import static io.trino.testing.StructuralTestUtil.sqlMapOf; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static java.nio.file.Files.createTempDirectory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; - -@TestInstance(PER_CLASS) -@Execution(CONCURRENT) -public class TestShardWriter -{ - private final Path directory; - - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - public TestShardWriter() - throws IOException - { - directory = createTempDirectory(null); - } - - @AfterAll - public void tearDown() - throws Exception - { - deleteRecursively(directory, ALLOW_INSECURE); - } - - @Test - public void testWriter() - throws Exception - { - List columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L); - ArrayType arrayType = new ArrayType(BIGINT); - ArrayType arrayOfArrayType = new ArrayType(arrayType); - Type mapType = TESTING_TYPE_MANAGER.getParameterizedType(StandardTypes.MAP, ImmutableList.of( - TypeSignatureParameter.typeParameter(createVarcharType(10).getTypeSignature()), - TypeSignatureParameter.typeParameter(BOOLEAN.getTypeSignature()))); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType); - File file = directory.resolve(System.nanoTime() + ".orc").toFile(); - - byte[] bytes1 = octets(0x00, 0xFE, 0xFF); - byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80); - - RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(columnTypes) - .row(123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))) - .row(null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), sqlMapOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))) - .row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT))); - - try (ThreadContextClassLoader _ = new ThreadContextClassLoader(new EmptyClassLoader()); - OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - writer.appendPages(rowPagesBuilder.build()); - } - - try (OrcDataSource dataSource = fileOrcDataSource(file)) { - OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); - assertThat(reader.getReaderRowCount()).isEqualTo(3); - assertThat(reader.getReaderPosition()).isEqualTo(0); - assertThat(reader.getFileRowCount()).isEqualTo(reader.getReaderRowCount()); - assertThat(reader.getFilePosition()).isEqualTo(reader.getFilePosition()); - - Page page = reader.nextPage(); - assertThat(page.getPositionCount()).isEqualTo(3); - assertThat(reader.getReaderPosition()).isEqualTo(0); - assertThat(reader.getFilePosition()).isEqualTo(reader.getFilePosition()); - - Block column0 = page.getBlock(0); - assertThat(column0.isNull(0)).isEqualTo(false); - assertThat(column0.isNull(1)).isEqualTo(true); - assertThat(column0.isNull(2)).isEqualTo(false); - assertThat(BIGINT.getLong(column0, 0)).isEqualTo(123L); - assertThat(BIGINT.getLong(column0, 2)).isEqualTo(456L); - - Block column1 = page.getBlock(1); - assertThat(createVarcharType(10).getSlice(column1, 0)).isEqualTo(utf8Slice("hello")); - assertThat(createVarcharType(10).getSlice(column1, 1)).isEqualTo(utf8Slice("world")); - assertThat(createVarcharType(10).getSlice(column1, 2)).isEqualTo(utf8Slice("bye \u2603")); - - Block column2 = page.getBlock(2); - assertThat(VARBINARY.getSlice(column2, 0)).isEqualTo(wrappedBuffer(bytes1)); - assertThat(column2.isNull(1)).isEqualTo(true); - assertThat(VARBINARY.getSlice(column2, 2)).isEqualTo(wrappedBuffer(bytes3)); - - Block column3 = page.getBlock(3); - assertThat(column3.isNull(0)).isEqualTo(false); - assertThat(column3.isNull(1)).isEqualTo(false); - assertThat(column3.isNull(2)).isEqualTo(false); - assertThat(DOUBLE.getDouble(column3, 0)).isEqualTo(123.456); - assertThat(DOUBLE.getDouble(column3, 1)).isEqualTo(Double.POSITIVE_INFINITY); - assertThat(DOUBLE.getDouble(column3, 2)).isNaN(); - - Block column4 = page.getBlock(4); - assertThat(column4.isNull(0)).isEqualTo(false); - assertThat(column4.isNull(1)).isEqualTo(true); - assertThat(column4.isNull(2)).isEqualTo(false); - assertThat(BOOLEAN.getBoolean(column4, 0)).isEqualTo(true); - assertThat(BOOLEAN.getBoolean(column4, 2)).isEqualTo(false); - - Block column5 = page.getBlock(5); - assertThat(column5.getPositionCount()).isEqualTo(3); - - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 0), arrayBlockOf(BIGINT, 1, 2))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 1), arrayBlockOf(BIGINT, 3, null))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 2), arrayBlockOf(BIGINT))).isTrue(); - - Block column6 = page.getBlock(6); - assertThat(column6.getPositionCount()).isEqualTo(3); - - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column6, 0), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true))).isTrue(); - SqlMap object = (SqlMap) mapType.getObject(column6, 1); - SqlMap k2 = sqlMapOf(createVarcharType(5), BOOLEAN, "k2", null); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, object, k2)).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column6, 2), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", false))).isTrue(); - - Block column7 = page.getBlock(7); - assertThat(column7.getPositionCount()).isEqualTo(3); - - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 1), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)))).isTrue(); - - assertThat(reader.nextPage()).isNull(); - assertThat(reader.getReaderPosition()).isEqualTo(3); - assertThat(reader.getFilePosition()).isEqualTo(reader.getFilePosition()); - - OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes()); - assertThat(orcFileMetadata).isEqualTo(new OrcFileMetadata(ImmutableMap.builder() - .put(1L, BIGINT.getTypeId()) - .put(2L, createVarcharType(10).getTypeId()) - .put(4L, VARBINARY.getTypeId()) - .put(6L, DOUBLE.getTypeId()) - .put(7L, BOOLEAN.getTypeId()) - .put(8L, arrayType.getTypeId()) - .put(9L, mapType.getTypeId()) - .put(10L, arrayOfArrayType.getTypeId()) - .buildOrThrow())); - } - - File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc"); - assertThat(crcFile).doesNotExist(); - } - - @SuppressWarnings("EmptyClass") - private static class EmptyClassLoader - extends ClassLoader - { - protected EmptyClassLoader() - { - super(null); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestStorageManagerConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestStorageManagerConfig.java deleted file mode 100644 index 17deeaeb059e..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestStorageManagerConfig.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import jakarta.validation.constraints.NotNull; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static io.airlift.testing.ValidationAssertions.assertFailsValidation; -import static io.airlift.units.DataSize.Unit.GIGABYTE; -import static io.airlift.units.DataSize.Unit.KILOBYTE; -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static java.lang.Math.max; -import static java.lang.Runtime.getRuntime; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class TestStorageManagerConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(StorageManagerConfig.class) - .setDataDirectory(null) - .setMinAvailableSpace(DataSize.ofBytes(0)) - .setOrcMaxMergeDistance(DataSize.of(1, MEGABYTE)) - .setOrcMaxReadSize(DataSize.of(8, MEGABYTE)) - .setOrcStreamBufferSize(DataSize.of(8, MEGABYTE)) - .setOrcTinyStripeThreshold(DataSize.of(8, MEGABYTE)) - .setOrcLazyReadSmallRanges(true) - .setOrcNestedLazy(true) - .setDeletionThreads(max(1, getRuntime().availableProcessors() / 2)) - .setShardRecoveryTimeout(new Duration(30, SECONDS)) - .setMissingShardDiscoveryInterval(new Duration(5, MINUTES)) - .setCompactionInterval(new Duration(1, HOURS)) - .setShardEjectorInterval(new Duration(4, HOURS)) - .setRecoveryThreads(10) - .setOrganizationThreads(5) - .setCompactionEnabled(true) - .setOrganizationEnabled(true) - .setOrganizationInterval(new Duration(7, DAYS)) - .setOrganizationDiscoveryInterval(new Duration(6, HOURS)) - .setMaxShardRows(1_000_000) - .setMaxShardSize(DataSize.of(256, MEGABYTE)) - .setMaxBufferSize(DataSize.of(256, MEGABYTE)) - .setOneSplitPerBucketThreshold(0)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("storage.data-directory", "/data") - .put("storage.min-available-space", "123GB") - .put("storage.orc.max-merge-distance", "16kB") - .put("storage.orc.max-read-size", "16kB") - .put("storage.orc.stream-buffer-size", "16kB") - .put("storage.orc.tiny-stripe-threshold", "15kB") - .put("storage.orc.lazy-read-small-ranges", "false") - .put("storage.orc.nested-lazy", "false") - .put("storage.max-deletion-threads", "999") - .put("storage.shard-recovery-timeout", "1m") - .put("storage.missing-shard-discovery-interval", "4m") - .put("storage.compaction-enabled", "false") - .put("storage.compaction-interval", "4h") - .put("storage.organization-enabled", "false") - .put("storage.organization-interval", "4h") - .put("storage.organization-discovery-interval", "2h") - .put("storage.ejector-interval", "9h") - .put("storage.max-recovery-threads", "12") - .put("storage.max-organization-threads", "12") - .put("storage.max-shard-rows", "10000") - .put("storage.max-shard-size", "10MB") - .put("storage.max-buffer-size", "512MB") - .put("storage.one-split-per-bucket-threshold", "4") - .buildOrThrow(); - - StorageManagerConfig expected = new StorageManagerConfig() - .setDataDirectory(new File("/data")) - .setMinAvailableSpace(DataSize.of(123, GIGABYTE)) - .setOrcMaxMergeDistance(DataSize.of(16, KILOBYTE)) - .setOrcMaxReadSize(DataSize.of(16, KILOBYTE)) - .setOrcStreamBufferSize(DataSize.of(16, KILOBYTE)) - .setOrcTinyStripeThreshold(DataSize.of(15, KILOBYTE)) - .setOrcLazyReadSmallRanges(false) - .setOrcNestedLazy(false) - .setDeletionThreads(999) - .setShardRecoveryTimeout(new Duration(1, MINUTES)) - .setMissingShardDiscoveryInterval(new Duration(4, MINUTES)) - .setCompactionEnabled(false) - .setCompactionInterval(new Duration(4, HOURS)) - .setOrganizationEnabled(false) - .setOrganizationInterval(new Duration(4, HOURS)) - .setOrganizationDiscoveryInterval(new Duration(2, HOURS)) - .setShardEjectorInterval(new Duration(9, HOURS)) - .setRecoveryThreads(12) - .setOrganizationThreads(12) - .setMaxShardRows(10_000) - .setMaxShardSize(DataSize.of(10, MEGABYTE)) - .setMaxBufferSize(DataSize.of(512, MEGABYTE)) - .setOneSplitPerBucketThreshold(4); - - assertFullMapping(properties, expected); - } - - @Test - public void testValidations() - { - assertFailsValidation(new StorageManagerConfig().setDataDirectory(null), "dataDirectory", "must not be null", NotNull.class); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestCompactionSetCreator.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestCompactionSetCreator.java deleted file mode 100644 index 149a0cfbaed4..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestCompactionSetCreator.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import java.time.Duration; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestCompactionSetCreator -{ - private static final long MAX_SHARD_ROWS = 100; - private static final DataSize MAX_SHARD_SIZE = DataSize.ofBytes(100); - private static final Table tableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.empty(), false); - private static final Table temporalTableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.of(1), false); - private static final Table bucketedTableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.of(3), OptionalLong.empty(), false); - private static final Table bucketedTemporalTableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.of(3), OptionalLong.of(1), false); - - private final CompactionSetCreator compactionSetCreator = new CompactionSetCreator(MAX_SHARD_SIZE, MAX_SHARD_ROWS); - - @Test - public void testNonTemporalOrganizationSetSimple() - { - List inputShards = ImmutableList.of( - shardWithSize(10, 10), - shardWithSize(10, 10), - shardWithSize(10, 10)); - - Set compactionSets = compactionSetCreator.createCompactionSets(tableInfo, inputShards); - assertThat(compactionSets.size()).isEqualTo(1); - assertThat(getOnlyElement(compactionSets).getShards()).isEqualTo(extractIndexes(inputShards, 0, 1, 2)); - } - - @Test - public void testNonTemporalSizeBasedOrganizationSet() - { - List inputShards = ImmutableList.of( - shardWithSize(10, 70), - shardWithSize(10, 20), - shardWithSize(10, 30), - shardWithSize(10, 120)); - - Set compactionSets = compactionSetCreator.createCompactionSets(tableInfo, inputShards); - - Set actual = new HashSet<>(); - for (OrganizationSet set : compactionSets) { - actual.addAll(set.getShards()); - } - assertThat(extractIndexes(inputShards, 0, 1, 2)).containsAll(actual); - } - - @Test - public void testNonTemporalRowCountBasedOrganizationSet() - { - List inputShards = ImmutableList.of( - shardWithSize(50, 10), - shardWithSize(100, 10), - shardWithSize(20, 10), - shardWithSize(30, 10)); - - Set compactionSets = compactionSetCreator.createCompactionSets(tableInfo, inputShards); - - Set actual = new HashSet<>(); - for (OrganizationSet set : compactionSets) { - actual.addAll(set.getShards()); - } - - assertThat(extractIndexes(inputShards, 0, 2, 3)).containsAll(actual); - } - - @Test - public void testTemporalCompactionNoCompactionAcrossDays() - { - long day1 = Duration.ofDays(Duration.ofNanos(System.nanoTime()).toDays()).toMillis(); - long day2 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 1).toMillis(); - long day3 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 2).toMillis(); - - List inputShards = ImmutableList.of( - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day1), - shardWithTemporalRange(TIMESTAMP_MILLIS, day2, day2), - shardWithTemporalRange(TIMESTAMP_MILLIS, day2, day2), - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day1), - shardWithTemporalRange(TIMESTAMP_MILLIS, day3, day3)); - - Set actual = compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards); - assertThat(actual.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(temporalTableInfo.getTableId(), extractIndexes(inputShards, 0, 3), OptionalInt.empty()), - new OrganizationSet(temporalTableInfo.getTableId(), extractIndexes(inputShards, 1, 2), OptionalInt.empty())); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testTemporalCompactionSpanningDays() - { - long day1 = Duration.ofDays(Duration.ofNanos(System.nanoTime()).toDays()).toMillis(); - long day2 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 1).toMillis(); - long day3 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 2).toMillis(); - long day4 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 3).toMillis(); - - List inputShards = ImmutableList.of( - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day3), // day2 - shardWithTemporalRange(TIMESTAMP_MILLIS, day2, day2), // day2 - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day1), // day1 - shardWithTemporalRange(TIMESTAMP_MILLIS, day1 + 100, day2 + 100), // day1 - shardWithTemporalRange(TIMESTAMP_MILLIS, day1 - 100, day2 - 100), // day1 - shardWithTemporalRange(TIMESTAMP_MILLIS, day2 - 100, day3 - 100), // day2 - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day4)); // day2 - - long tableId = temporalTableInfo.getTableId(); - Set compactionSets = compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards); - - assertThat(compactionSets.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(tableId, extractIndexes(inputShards, 0, 1, 5, 6), OptionalInt.empty()), - new OrganizationSet(tableId, extractIndexes(inputShards, 2, 3, 4), OptionalInt.empty())); - assertThat(compactionSets).isEqualTo(expected); - } - - @Test - public void testTemporalCompactionDate() - { - long day1 = Duration.ofNanos(System.nanoTime()).toDays(); - long day2 = day1 + 1; - long day3 = day1 + 2; - - List inputShards = ImmutableList.of( - shardWithTemporalRange(DATE, day1, day1), - shardWithTemporalRange(DATE, day2, day2), - shardWithTemporalRange(DATE, day3, day3), - shardWithTemporalRange(DATE, day1, day3), - shardWithTemporalRange(DATE, day2, day3), - shardWithTemporalRange(DATE, day1, day2)); - - long tableId = temporalTableInfo.getTableId(); - Set actual = compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards); - - assertThat(actual.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(tableId, extractIndexes(inputShards, 0, 3, 5), OptionalInt.empty()), - new OrganizationSet(tableId, extractIndexes(inputShards, 1, 4), OptionalInt.empty())); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testBucketedTableCompaction() - { - List inputShards = ImmutableList.of( - shardWithBucket(1), - shardWithBucket(2), - shardWithBucket(2), - shardWithBucket(1), - shardWithBucket(2), - shardWithBucket(1)); - - long tableId = bucketedTableInfo.getTableId(); - Set actual = compactionSetCreator.createCompactionSets(bucketedTableInfo, inputShards); - - assertThat(actual.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(tableId, extractIndexes(inputShards, 0, 3, 5), OptionalInt.of(1)), - new OrganizationSet(tableId, extractIndexes(inputShards, 1, 2, 4), OptionalInt.of(2))); - assertThat(actual).isEqualTo(expected); - } - - static Set extractIndexes(List inputShards, int... indexes) - { - ImmutableSet.Builder builder = ImmutableSet.builder(); - for (int index : indexes) { - builder.add(inputShards.get(index).getShardUuid()); - } - return builder.build(); - } - - @Test - public void testBucketedTemporalTableCompaction() - { - long day1 = 1; - long day2 = 2; - long day3 = 3; - long day4 = 4; - - List inputShards = ImmutableList.of( - shardWithTemporalBucket(OptionalInt.of(1), DATE, day1, day1), - shardWithTemporalBucket(OptionalInt.of(2), DATE, day2, day2), - shardWithTemporalBucket(OptionalInt.of(1), DATE, day1, day1), - shardWithTemporalBucket(OptionalInt.of(2), DATE, day2, day2), - shardWithTemporalBucket(OptionalInt.of(1), DATE, day3, day3), - shardWithTemporalBucket(OptionalInt.of(2), DATE, day4, day4)); - - long tableId = bucketedTemporalTableInfo.getTableId(); - Set actual = compactionSetCreator.createCompactionSets(bucketedTemporalTableInfo, inputShards); - - assertThat(actual.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(tableId, extractIndexes(inputShards, 0, 2), OptionalInt.of(1)), - new OrganizationSet(tableId, extractIndexes(inputShards, 1, 3), OptionalInt.of(2))); - assertThat(actual).isEqualTo(expected); - } - - private static ShardIndexInfo shardWithSize(long rows, long size) - { - return new ShardIndexInfo( - 1, - OptionalInt.empty(), - UUID.randomUUID(), - rows, - size, - Optional.empty(), - Optional.empty()); - } - - private static ShardIndexInfo shardWithTemporalRange(Type type, Long start, Long end) - { - return shardWithTemporalBucket(OptionalInt.empty(), type, start, end); - } - - private static ShardIndexInfo shardWithBucket(int bucketNumber) - { - return new ShardIndexInfo( - 1, - OptionalInt.of(bucketNumber), - UUID.randomUUID(), - 1, - 1, - Optional.empty(), - Optional.empty()); - } - - private static ShardIndexInfo shardWithTemporalBucket(OptionalInt bucketNumber, Type type, Long start, Long end) - { - if (type.equals(DATE)) { - return new ShardIndexInfo( - 1, - bucketNumber, - UUID.randomUUID(), - 1, - 1, - Optional.empty(), - Optional.of(ShardRange.of(new Tuple(type, start.intValue()), new Tuple(type, end.intValue())))); - } - return new ShardIndexInfo( - 1, - bucketNumber, - UUID.randomUUID(), - 1, - 1, - Optional.empty(), - Optional.of(ShardRange.of(new Tuple(type, start), new Tuple(type, end)))); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java deleted file mode 100644 index daf49460b080..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import io.airlift.units.DataSize; -import io.trino.SequencePageBuilder; -import io.trino.operator.PagesIndex; -import io.trino.operator.PagesIndexPageSorter; -import io.trino.orc.OrcReaderOptions; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.RaptorStorageManager; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.StoragePageSink; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeOperators; -import io.trino.testing.MaterializedResult; -import io.trino.testing.MaterializedRow; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.storage.TestRaptorStorageManager.createRaptorStorageManager; -import static io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.MaterializedResult.materializeSourceDataStream; -import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static java.nio.file.Files.createTempDirectory; -import static java.util.Collections.nCopies; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardCompactor -{ - private static final int MAX_SHARD_ROWS = 1000; - private static final PagesIndexPageSorter PAGE_SORTER = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); - private static final OrcReaderOptions READER_OPTIONS = new OrcReaderOptions() - .withMaxMergeDistance(DataSize.of(1, MEGABYTE)) - .withMaxBufferSize(DataSize.of(1, MEGABYTE)) - .withStreamBufferSize(DataSize.of(1, MEGABYTE)) - .withTinyStripeThreshold(DataSize.of(1, MEGABYTE)); - - private RaptorStorageManager storageManager; - private ShardCompactor compactor; - private Path temporary; - private Handle dummyHandle; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - storageManager = createRaptorStorageManager(dbi, temporary.toFile(), MAX_SHARD_ROWS); - compactor = new ShardCompactor(storageManager, READER_OPTIONS, new TypeOperators()); - } - - @AfterEach - public void tearDown() - throws Exception - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testShardCompactor() - throws Exception - { - List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP_MILLIS); - - List inputShards = createShards(storageManager, columnIds, columnTypes, 3); - assertThat(inputShards.size()).isEqualTo(3); - - long totalRows = inputShards.stream() - .mapToLong(ShardInfo::getRowCount) - .sum(); - long expectedOutputShards = computeExpectedOutputShards(totalRows); - - Set inputUuids = inputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); - - long transactionId = 1; - List outputShards = compactor.compact(transactionId, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes)); - assertThat(outputShards.size()).isEqualTo(expectedOutputShards); - - Set outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); - assertShardEqualsIgnoreOrder(inputUuids, outputUuids, columnIds, columnTypes); - } - - @Test - public void testShardCompactorSorted() - throws Exception - { - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DATE, TIMESTAMP_MILLIS, DOUBLE); - List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); - List sortColumnIds = ImmutableList.of(1L, 2L, 3L, 5L, 7L); - List sortOrders = nCopies(sortColumnIds.size(), ASC_NULLS_FIRST); - List sortIndexes = sortColumnIds.stream() - .map(columnIds::indexOf) - .collect(toList()); - - List inputShards = createSortedShards(storageManager, columnIds, columnTypes, sortIndexes, sortOrders, 2); - assertThat(inputShards.size()).isEqualTo(2); - - long totalRows = inputShards.stream().mapToLong(ShardInfo::getRowCount).sum(); - long expectedOutputShards = computeExpectedOutputShards(totalRows); - - Set inputUuids = inputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); - - long transactionId = 1; - List outputShards = compactor.compactSorted(transactionId, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes), sortColumnIds, sortOrders); - List outputUuids = outputShards.stream() - .map(ShardInfo::getShardUuid) - .collect(toList()); - assertThat(outputShards.size()).isEqualTo(expectedOutputShards); - - assertShardEqualsSorted(inputUuids, outputUuids, columnIds, columnTypes, sortIndexes, sortOrders); - } - - private static long computeExpectedOutputShards(long totalRows) - { - return ((totalRows % MAX_SHARD_ROWS) != 0) ? ((totalRows / MAX_SHARD_ROWS) + 1) : (totalRows / MAX_SHARD_ROWS); - } - - private void assertShardEqualsIgnoreOrder(Set inputUuids, Set outputUuids, List columnIds, List columnTypes) - throws IOException - { - MaterializedResult inputRows = getMaterializedRows(ImmutableList.copyOf(inputUuids), columnIds, columnTypes); - MaterializedResult outputRows = getMaterializedRows(ImmutableList.copyOf(outputUuids), columnIds, columnTypes); - - assertEqualsIgnoreOrder(outputRows, inputRows); - } - - private void assertShardEqualsSorted(Set inputUuids, List outputUuids, List columnIds, List columnTypes, List sortIndexes, List sortOrders) - throws IOException - { - List inputPages = getPages(inputUuids, columnIds, columnTypes); - List sortTypes = sortIndexes.stream().map(columnTypes::get).collect(toList()); - - MaterializedResult inputRowsSorted = sortAndMaterialize(inputPages, columnTypes, sortIndexes, sortOrders, sortTypes); - MaterializedResult outputRows = extractColumns(getMaterializedRows(outputUuids, columnIds, columnTypes), sortIndexes, sortTypes); - - assertThat(outputRows).containsExactlyElementsOf(inputRowsSorted); - } - - private static MaterializedResult extractColumns(MaterializedResult materializedRows, List indexes, List types) - { - ImmutableList.Builder rows = ImmutableList.builder(); - for (MaterializedRow row : materializedRows) { - Object[] values = new Object[indexes.size()]; - for (int i = 0; i < indexes.size(); i++) { - values[i] = row.getField(indexes.get(i)); - } - rows.add(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, values)); - } - return new MaterializedResult(rows.build(), types); - } - - private static MaterializedResult sortAndMaterialize(List pages, List columnTypes, List sortIndexes, List sortOrders, List sortTypes) - { - long[] orderedAddresses = PAGE_SORTER.sort(columnTypes, pages, sortIndexes, sortOrders, 10_000); - - PageBuilder pageBuilder = new PageBuilder(columnTypes); - for (long orderedAddress : orderedAddresses) { - int pageIndex = PAGE_SORTER.decodePageIndex(orderedAddress); - int positionIndex = PAGE_SORTER.decodePositionIndex(orderedAddress); - - Page page = pages.get(pageIndex); - pageBuilder.declarePosition(); - for (int i = 0; i < columnTypes.size(); i++) { - columnTypes.get(i).appendTo(page.getBlock(i), positionIndex, pageBuilder.getBlockBuilder(i)); - } - } - - // extract the sortIndexes and reorder the blocks by sort indexes (useful for debugging) - Page buildPage = pageBuilder.build(); - Block[] outputBlocks = new Block[buildPage.getChannelCount()]; - - for (int i = 0; i < sortIndexes.size(); i++) { - outputBlocks[i] = buildPage.getBlock(sortIndexes.get(i)); - } - - MaterializedResult.Builder resultBuilder = MaterializedResult.resultBuilder(SESSION, sortTypes); - resultBuilder.page(new Page(outputBlocks)); - - return resultBuilder.build(); - } - - private List getPages(Set uuids, List columnIds, List columnTypes) - throws IOException - { - ImmutableList.Builder pages = ImmutableList.builder(); - for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = getPageSource(columnIds, columnTypes, uuid)) { - while (!pageSource.isFinished()) { - Page outputPage = pageSource.getNextPage(); - if (outputPage == null) { - break; - } - pages.add(outputPage.getLoadedPage()); - } - } - } - return pages.build(); - } - - private MaterializedResult getMaterializedRows(List uuids, List columnIds, List columnTypes) - throws IOException - { - MaterializedResult.Builder rows = MaterializedResult.resultBuilder(SESSION, columnTypes); - for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = getPageSource(columnIds, columnTypes, uuid)) { - MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); - rows.rows(result.getMaterializedRows()); - } - } - return rows.build(); - } - - private ConnectorPageSource getPageSource(List columnIds, List columnTypes, UUID uuid) - { - return storageManager.getPageSource(uuid, OptionalInt.empty(), columnIds, columnTypes, TupleDomain.all(), READER_OPTIONS); - } - - private static List createSortedShards(StorageManager storageManager, List columnIds, List columnTypes, List sortChannels, List sortOrders, int shardCount) - { - StoragePageSink sink = createStoragePageSink(storageManager, columnIds, columnTypes); - for (int shardNum = 0; shardNum < shardCount; shardNum++) { - createSortedShard(columnTypes, sortChannels, sortOrders, sink); - } - return getFutureValue(sink.commit()); - } - - private static void createSortedShard(List columnTypes, List sortChannels, List sortOrders, StoragePageSink sink) - { - List pages = createPages(columnTypes); - - // Sort pages - long[] orderedAddresses = PAGE_SORTER.sort(columnTypes, pages, sortChannels, sortOrders, 10_000); - int[] orderedPageIndex = new int[orderedAddresses.length]; - int[] orderedPositionIndex = new int[orderedAddresses.length]; - - for (int i = 0; i < orderedAddresses.length; i++) { - orderedPageIndex[i] = PAGE_SORTER.decodePageIndex(orderedAddresses[i]); - orderedPositionIndex[i] = PAGE_SORTER.decodePositionIndex(orderedAddresses[i]); - } - - // Append sorted pages - sink.appendPages(pages, orderedPageIndex, orderedPositionIndex); - sink.flush(); - } - - private static List createShards(StorageManager storageManager, List columnIds, List columnTypes, int shardCount) - { - StoragePageSink sink = createStoragePageSink(storageManager, columnIds, columnTypes); - for (int i = 0; i < shardCount; i++) { - sink.appendPages(createPages(columnTypes)); - sink.flush(); - } - return getFutureValue(sink.commit()); - } - - private static StoragePageSink createStoragePageSink(StorageManager manager, List columnIds, List columnTypes) - { - long transactionId = 1; - return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); - } - - private static List createPages(List columnTypes) - { - // Creates 10 pages with 10 rows each - int rowCount = 10; - int pageCount = 10; - - // some random values to start off the blocks - int[][] initialValues = {{17, 15, 16, 18, 14}, {59, 55, 54, 53, 58}}; - - ImmutableList.Builder pages = ImmutableList.builder(); - for (int i = 0; i < pageCount; i++) { - pages.add(SequencePageBuilder.createSequencePage(columnTypes, rowCount, initialValues[i % 2])); - } - return pages.build(); - } - - private static List getColumnInfo(List columnIds, List columnTypes) - { - ImmutableList.Builder columnInfos = ImmutableList.builder(); - for (int i = 0; i < columnIds.size(); i++) { - columnInfos.add(new ColumnInfo(columnIds.get(i), columnTypes.get(i))); - } - return columnInfos.build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizationManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizationManager.java deleted file mode 100644 index 7ee6ae0061a6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizationManager.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizationManager.createOrganizationSets; -import static io.trino.plugin.raptor.legacy.storage.organization.TestCompactionSetCreator.extractIndexes; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardOrganizationManager -{ - private Jdbi dbi; - private Handle dummyHandle; - private MetadataDao metadataDao; - private ShardOrganizerDao organizerDao; - - private static final Table tableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.empty(), true); - private static final Table temporalTableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.of(1), true); - - private static final List types = ImmutableList.of(BIGINT, VARCHAR, DATE, TIMESTAMP_MILLIS); - - @BeforeEach - public void setup() - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - metadataDao = dbi.onDemand(MetadataDao.class); - organizerDao = dbi.onDemand(ShardOrganizerDao.class); - - createTablesWithRetry(dbi); - } - - @AfterEach - public void teardown() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testOrganizationEligibleTables() - { - long table1 = metadataDao.insertTable("schema", "table1", false, true, null, 0); - metadataDao.insertColumn(table1, 1, "foo", 1, "bigint", 1, null); - - metadataDao.insertTable("schema", "table2", false, true, null, 0); - metadataDao.insertTable("schema", "table3", false, false, null, 0); - assertThat(metadataDao.getOrganizationEligibleTables()).isEqualTo(ImmutableSet.of(table1)); - } - - @Test - public void testTableDiscovery() - throws Exception - { - long table1 = metadataDao.insertTable("schema", "table1", false, true, null, 0); - metadataDao.insertColumn(table1, 1, "foo", 1, "bigint", 1, null); - - long table2 = metadataDao.insertTable("schema", "table2", false, true, null, 0); - metadataDao.insertColumn(table2, 1, "foo", 1, "bigint", 1, null); - - metadataDao.insertTable("schema", "table3", false, false, null, 0); - - long intervalMillis = 100; - ShardOrganizationManager organizationManager = createShardOrganizationManager(intervalMillis); - - // initializes tables - Set actual = organizationManager.discoverAndInitializeTablesToOrganize(); - assertThat(actual).isEqualTo(ImmutableSet.of(table1, table2)); - - // update the start times and test that the tables are discovered after interval seconds - long updateTime = System.currentTimeMillis(); - organizerDao.updateLastStartTime("node1", table1, updateTime); - organizerDao.updateLastStartTime("node1", table2, updateTime); - - // wait for some time (interval time) for the tables to be eligible for organization - long start = System.nanoTime(); - while (organizationManager.discoverAndInitializeTablesToOrganize().isEmpty() && - nanosSince(start).toMillis() < intervalMillis + 1000) { - MILLISECONDS.sleep(10); - } - assertThat(organizationManager.discoverAndInitializeTablesToOrganize()).isEqualTo(ImmutableSet.of(table1, table2)); - } - - @Test - public void testSimple() - { - long timestamp = 1L; - int day = 1; - - List shards = ImmutableList.of( - shardWithSortRange(1, ShardRange.of(new Tuple(types, 5L, "hello", day, timestamp), new Tuple(types, 10L, "hello", day, timestamp))), - shardWithSortRange(1, ShardRange.of(new Tuple(types, 7L, "hello", day, timestamp), new Tuple(types, 10L, "hello", day, timestamp))), - shardWithSortRange(1, ShardRange.of(new Tuple(types, 6L, "hello", day, timestamp), new Tuple(types, 9L, "hello", day, timestamp))), - shardWithSortRange(1, ShardRange.of(new Tuple(types, 1L, "hello", day, timestamp), new Tuple(types, 5L, "hello", day, timestamp)))); - Set actual = createOrganizationSets(tableInfo, shards); - - assertThat(actual.size()).isEqualTo(1); - // Shards 0, 1 and 2 are overlapping, so we should get an organization set with these shards - assertThat(getOnlyElement(actual).getShards()).isEqualTo(extractIndexes(shards, 0, 1, 2)); - } - - @Test - public void testSimpleTemporal() - { - List temporalType = ImmutableList.of(DATE); - List types = ImmutableList.of(BIGINT); - - int day1 = 1; - int day2 = 2; - int day4 = 4; - int day5 = 5; - - List shards = ImmutableList.of( - shardWithTemporalRange(1, ShardRange.of(new Tuple(types, 5L), new Tuple(types, 10L)), ShardRange.of(new Tuple(temporalType, day1), new Tuple(temporalType, day2))), - shardWithTemporalRange(1, ShardRange.of(new Tuple(types, 7L), new Tuple(types, 10L)), ShardRange.of(new Tuple(temporalType, day4), new Tuple(temporalType, day5))), - shardWithTemporalRange(1, ShardRange.of(new Tuple(types, 6L), new Tuple(types, 9L)), ShardRange.of(new Tuple(temporalType, day1), new Tuple(temporalType, day2))), - shardWithTemporalRange(1, ShardRange.of(new Tuple(types, 4L), new Tuple(types, 8L)), ShardRange.of(new Tuple(temporalType, day4), new Tuple(temporalType, day5)))); - - Set organizationSets = createOrganizationSets(temporalTableInfo, shards); - Set> actual = organizationSets.stream() - .map(OrganizationSet::getShards) - .collect(toSet()); - - // expect 2 organization sets, of overlapping shards (0, 2) and (1, 3) - assertThat(organizationSets.size()).isEqualTo(2); - assertThat(actual).isEqualTo(ImmutableSet.of(extractIndexes(shards, 0, 2), extractIndexes(shards, 1, 3))); - } - - private static ShardIndexInfo shardWithSortRange(int bucketNumber, ShardRange sortRange) - { - return new ShardIndexInfo( - 1, - OptionalInt.of(bucketNumber), - UUID.randomUUID(), - 1, - 1, - Optional.of(sortRange), - Optional.empty()); - } - - private static ShardIndexInfo shardWithTemporalRange(int bucketNumber, ShardRange sortRange, ShardRange temporalRange) - { - return new ShardIndexInfo( - 1, - OptionalInt.of(bucketNumber), - UUID.randomUUID(), - 1, - 1, - Optional.of(sortRange), - Optional.of(temporalRange)); - } - - private ShardOrganizationManager createShardOrganizationManager(long intervalMillis) - { - return new ShardOrganizationManager(dbi, - "node1", - createShardManager(dbi), - createShardOrganizer(), - true, - new Duration(intervalMillis, MILLISECONDS), - new Duration(5, MINUTES)); - } - - private static class MockJobFactory - implements JobFactory - { - @Override - public Runnable create(OrganizationSet organizationSet) - { - return () -> sleepUninterruptibly(10, MILLISECONDS); - } - } - - private static ShardOrganizer createShardOrganizer() - { - return new ShardOrganizer(new MockJobFactory(), 1); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizer.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizer.java deleted file mode 100644 index f72f4f464a55..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizer.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableSet; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestShardOrganizer -{ - @Test - @Timeout(5) - public void testShardOrganizerInProgress() - throws Exception - { - CountDownLatch canComplete = new CountDownLatch(1); - ShardOrganizer organizer = new ShardOrganizer( - organizationSet -> () -> checkState(awaitUninterruptibly(canComplete, 10, SECONDS)), - 1); - - Set shards = ImmutableSet.of(UUID.randomUUID()); - OrganizationSet organizationSet = new OrganizationSet(1L, shards, OptionalInt.empty()); - - organizer.enqueue(organizationSet); - - assertThat(organizer.inProgress(getOnlyElement(shards))).isTrue(); - assertThat(organizer.getShardsInProgress()).isEqualTo(1); - - canComplete.countDown(); - while (organizer.inProgress(getOnlyElement(shards))) { - MILLISECONDS.sleep(10); - } - assertThat(organizer.inProgress(getOnlyElement(shards))).isFalse(); - assertThat(organizer.getShardsInProgress()).isEqualTo(0); - organizer.shutdown(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizerUtil.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizerUtil.java deleted file mode 100644 index fcb49f441cb0..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizerUtil.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import io.trino.plugin.raptor.legacy.RaptorMetadata; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.ColumnStats; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.nio.file.Files; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.collect.MoreCollectors.onlyElement; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.shardInfo; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getOrganizationEligibleShards; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardOrganizerUtil -{ - private static final List COLUMNS = ImmutableList.of( - new ColumnInfo(1, TIMESTAMP_MILLIS), - new ColumnInfo(2, BIGINT), - new ColumnInfo(3, VARCHAR)); - - private Jdbi dbi; - private Handle dummyHandle; - private File dataDir; - private ShardManager shardManager; - private MetadataDao metadataDao; - private ConnectorMetadata metadata; - - @BeforeEach - public void setup() - throws Exception - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - dataDir = Files.createTempDirectory(null).toFile(); - - metadata = new RaptorMetadata(dbi, createShardManager(dbi)); - - metadataDao = dbi.onDemand(MetadataDao.class); - shardManager = createShardManager(dbi); - } - - @AfterEach - public void teardown() - throws Exception - { - dummyHandle.close(); - dummyHandle = null; - deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); - } - - @Test - public void testGetOrganizationEligibleShards() - { - int day1 = 1111; - int day2 = 2222; - - SchemaTableName tableName = new SchemaTableName("default", "test"); - metadata.createTable(SESSION, tableMetadataBuilder(tableName) - .column("orderkey", BIGINT) - .column("orderdate", DATE) - .column("orderstatus", createVarcharType(3)) - .property("ordering", ImmutableList.of("orderstatus", "orderkey")) - .property("temporal_column", "orderdate") - .build(), - SaveMode.FAIL); - Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName()); - List tableColumns = metadataDao.listTableColumns(tableInfo.getTableId()); - Map tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName); - - long orderDate = tableColumnMap.get("orderdate").getColumnId(); - long orderKey = tableColumnMap.get("orderkey").getColumnId(); - long orderStatus = tableColumnMap.get("orderstatus").getColumnId(); - - List shards = ImmutableList.builder() - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day1, day1 + 10), - new ColumnStats(orderKey, 13L, 14L), - new ColumnStats(orderStatus, "aaa", "abc")))) - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day2, day2 + 100), - new ColumnStats(orderKey, 2L, 20L), - new ColumnStats(orderStatus, "aaa", "abc")))) - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day1, day2), - new ColumnStats(orderKey, 2L, 11L), - new ColumnStats(orderStatus, "aaa", "abc")))) - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day1, day2), - new ColumnStats(orderKey, 2L, null), - new ColumnStats(orderStatus, "aaa", "abc")))) - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day1, null), - new ColumnStats(orderKey, 2L, 11L), - new ColumnStats(orderStatus, "aaa", "abc")))) - .build(); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0); - Set shardMetadatas = shardManager.getNodeShards("node1"); - - Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId()); - TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId); - - Set actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false)); - List expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty()); - - assertThat(actual).containsExactlyElementsOf(expected); - - List sortColumns = metadataDao.listSortColumns(tableInfo.getTableId()); - Set actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true)); - List expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns)); - - assertThat(actualSortRange).containsExactlyElementsOf(expectedSortRange); - } - - private static List getShardIndexInfo(Table tableInfo, List shards, TableColumn temporalColumn, Optional> sortColumns) - { - long tableId = tableInfo.getTableId(); - Type temporalType = temporalColumn.getDataType(); - - ImmutableList.Builder builder = ImmutableList.builder(); - for (ShardInfo shard : shards) { - ColumnStats temporalColumnStats = shard.getColumnStats().stream() - .filter(columnStats -> columnStats.getColumnId() == temporalColumn.getColumnId()) - .collect(onlyElement()); - - if (temporalColumnStats.getMin() == null || temporalColumnStats.getMax() == null) { - continue; - } - - Optional sortRange = Optional.empty(); - if (sortColumns.isPresent()) { - Map columnIdToStats = Maps.uniqueIndex(shard.getColumnStats(), ColumnStats::getColumnId); - ImmutableList.Builder typesBuilder = ImmutableList.builder(); - ImmutableList.Builder minBuilder = ImmutableList.builder(); - ImmutableList.Builder maxBuilder = ImmutableList.builder(); - boolean isShardEligible = true; - for (TableColumn sortColumn : sortColumns.get()) { - ColumnStats columnStats = columnIdToStats.get(sortColumn.getColumnId()); - typesBuilder.add(sortColumn.getDataType()); - - if (columnStats.getMin() == null || columnStats.getMax() == null) { - isShardEligible = false; - break; - } - - minBuilder.add(columnStats.getMin()); - maxBuilder.add(columnStats.getMax()); - } - - if (!isShardEligible) { - continue; - } - - List types = typesBuilder.build(); - List minValues = minBuilder.build(); - List maxValues = maxBuilder.build(); - sortRange = Optional.of(ShardRange.of(new Tuple(types, minValues), new Tuple(types, maxValues))); - } - builder.add(new ShardIndexInfo( - tableId, - OptionalInt.empty(), - shard.getShardUuid(), - shard.getRowCount(), - shard.getUncompressedSize(), - sortRange, - Optional.of(ShardRange.of( - new Tuple(temporalType, temporalColumnStats.getMin()), - new Tuple(temporalType, temporalColumnStats.getMax()))))); - } - return builder.build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardRange.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardRange.java deleted file mode 100644 index a85d24c2cb23..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardRange.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestShardRange -{ - @Test - public void testEnclosesIsSymmetric() - { - List types = ImmutableList.of(BIGINT, VARCHAR, BOOLEAN, TIMESTAMP_MILLIS); - ShardRange range = ShardRange.of(new Tuple(types, 2L, "aaa", true, 1L), new Tuple(types, 5L, "ccc", false, 2L)); - assertThat(range.encloses(range)).isTrue(); - } - - @Test - public void testEnclosingRange() - { - List types1 = ImmutableList.of(BIGINT); - ShardRange range1 = ShardRange.of(new Tuple(types1, 2L), new Tuple(types1, 5L)); - - ShardRange enclosesRange1 = ShardRange.of(new Tuple(types1, 1L), new Tuple(types1, 10L)); - ShardRange notEnclosesRange1 = ShardRange.of(new Tuple(types1, 1L), new Tuple(types1, 4L)); - - assertThat(enclosesRange1.encloses(range1)).isTrue(); - assertThat(notEnclosesRange1.encloses(range1)).isFalse(); - - List types2 = ImmutableList.of(BIGINT, VARCHAR); - ShardRange range2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "ccc")); - ShardRange enclosesRange2 = ShardRange.of(new Tuple(types2, 1L, "ccc"), new Tuple(types2, 10L, "ccc")); - ShardRange notEnclosesRange2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "bbb")); - - assertThat(range2.encloses(range2)).isTrue(); - assertThat(enclosesRange2.encloses(range2)).isTrue(); - assertThat(notEnclosesRange2.encloses(range2)).isFalse(); - } - - @Test - public void testOverlapsIsSymmetric() - { - List types = ImmutableList.of(BIGINT, VARCHAR, BOOLEAN, TIMESTAMP_MILLIS); - ShardRange range = ShardRange.of(new Tuple(types, 2L, "aaa", true, 1L), new Tuple(types, 5L, "ccc", false, 2L)); - assertThat(range.overlaps(range)).isTrue(); - } - - @Test - public void testOverlappingRange() - { - List types1 = ImmutableList.of(BIGINT); - ShardRange range1 = ShardRange.of(new Tuple(types1, 2L), new Tuple(types1, 5L)); - - ShardRange enclosesRange1 = ShardRange.of(new Tuple(types1, 1L), new Tuple(types1, 10L)); - ShardRange overlapsRange1 = ShardRange.of(new Tuple(types1, 1L), new Tuple(types1, 4L)); - ShardRange notOverlapsRange1 = ShardRange.of(new Tuple(types1, 6L), new Tuple(types1, 8L)); - - assertThat(enclosesRange1.overlaps(range1)).isTrue(); - assertThat(overlapsRange1.overlaps(range1)).isTrue(); - assertThat(notOverlapsRange1.overlaps(range1)).isFalse(); - - List types2 = ImmutableList.of(BIGINT, VARCHAR); - ShardRange range2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "ccc")); - ShardRange enclosesRange2 = ShardRange.of(new Tuple(types2, 1L, "ccc"), new Tuple(types2, 10L, "ccc")); - ShardRange overlapsRange2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "bbb")); - ShardRange notOverlapsRange2 = ShardRange.of(new Tuple(types2, 6L, "aaa"), new Tuple(types2, 8L, "bbb")); - - assertThat(enclosesRange2.encloses(range2)).isTrue(); - assertThat(overlapsRange2.overlaps(range2)).isTrue(); - assertThat(notOverlapsRange2.overlaps(range2)).isFalse(); - } - - @Test - public void testAdjacentRange() - { - List types1 = ImmutableList.of(BIGINT); - ShardRange range1 = ShardRange.of(new Tuple(types1, 2L), new Tuple(types1, 5L)); - ShardRange adjacentRange1 = ShardRange.of(new Tuple(types1, 5L), new Tuple(types1, 10L)); - - assertThat(range1.adjacent(range1)).isFalse(); - - assertThat(adjacentRange1.adjacent(range1)).isTrue(); - assertThat(range1.adjacent(adjacentRange1)).isTrue(); - - List types2 = ImmutableList.of(BIGINT, VARCHAR); - ShardRange range2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "ccc")); - ShardRange adjacentRange2 = ShardRange.of(new Tuple(types2, 5L, "ccc"), new Tuple(types2, 10L, "ccc")); - ShardRange subsetAdjacentRange2 = ShardRange.of(new Tuple(types2, 5L, "ddd"), new Tuple(types2, 10L, "ccc")); - ShardRange overlapsRange2 = ShardRange.of(new Tuple(types2, 3L, "aaa"), new Tuple(types2, 10L, "ccc")); - ShardRange notAdjacentRange2 = ShardRange.of(new Tuple(types2, 6L, "ccc"), new Tuple(types2, 10L, "ccc")); - - assertThat(adjacentRange2.adjacent(range2)).isTrue(); - assertThat(subsetAdjacentRange2.adjacent(range2)).isTrue(); - assertThat(overlapsRange2.adjacent(range2)).isFalse(); - assertThat(notAdjacentRange2.adjacent(range2)).isFalse(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTemporalFunction.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTemporalFunction.java deleted file mode 100644 index e294f9f51d8a..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTemporalFunction.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import org.joda.time.DateTime; -import org.junit.jupiter.api.Test; - -import java.time.Duration; - -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.type.DateTimes.MICROSECONDS_PER_MILLISECOND; -import static org.assertj.core.api.Assertions.assertThat; -import static org.joda.time.DateTimeZone.UTC; - -public class TestTemporalFunction -{ - private static final DateTime DATE_TIME = new DateTime(1970, 1, 2, 0, 0, 0, UTC); - - @Test - public void testDateBlock() - { - BlockBuilder blockBuilder = DATE.createBlockBuilder(null, 2); - DATE.writeLong(blockBuilder, 13); - DATE.writeLong(blockBuilder, 42); - Block block = blockBuilder.build(); - - assertThat(TemporalFunction.getDay(DATE, block, 0)).isEqualTo(13); - assertThat(TemporalFunction.getDay(DATE, block, 1)).isEqualTo(42); - } - - @Test - public void testTimestampBlock() - { - BlockBuilder blockBuilder = TIMESTAMP_MILLIS.createBlockBuilder(null, 4); - - // start and end of UTC day - TIMESTAMP_MILLIS.writeLong(blockBuilder, DATE_TIME.getMillis() * MICROSECONDS_PER_MILLISECOND); - TIMESTAMP_MILLIS.writeLong(blockBuilder, (DATE_TIME.getMillis() + Duration.ofHours(23).toMillis()) * MICROSECONDS_PER_MILLISECOND); - - Block block = blockBuilder.build(); - - assertThat(TemporalFunction.getDay(TIMESTAMP_MILLIS, block, 0)).isEqualTo(1); - assertThat(TemporalFunction.getDay(TIMESTAMP_MILLIS, block, 1)).isEqualTo(1); - } - - @Test - public void testDateShardRange() - { - assertThat(TemporalFunction.getDayFromRange(dateRange(2, 2))).isEqualTo(2); - assertThat(TemporalFunction.getDayFromRange(dateRange(13, 13))).isEqualTo(13); - - // date is determined from lowest shard - assertThat(TemporalFunction.getDayFromRange(dateRange(2, 5))).isEqualTo(2); - } - - @Test - public void testTimestampShardRange() - { - // The time frame should be look like following: - - // time range covers full day of day 1 - assertThat(TemporalFunction.getDayFromRange(timeRange(DATE_TIME.getMillis(), Duration.ofDays(1)))).isEqualTo(1); - // time range covers full day of day 1 and 2 - assertThat(TemporalFunction.getDayFromRange(timeRange(DATE_TIME.getMillis(), Duration.ofDays(2)))).isEqualTo(2); - // time range covers 13 hours of day 1 and 11 hours of day 2 - assertThat(TemporalFunction.getDayFromRange(timeRange(DATE_TIME.getMillis() + Duration.ofHours(11).toMillis(), Duration.ofHours(24)))).isEqualTo(1); - // time range covers 11 hours of day 0 and 13 hours of day 1 - assertThat(TemporalFunction.getDayFromRange(timeRange(DATE_TIME.getMillis() + Duration.ofHours(13).toMillis(), Duration.ofHours(24)))).isEqualTo(2); - } - - private static ShardRange dateRange(int start, int end) - { - return ShardRange.of(new Tuple(DATE, start), new Tuple(DATE, end)); - } - - private static ShardRange timeRange(long start, Duration duration) - { - return ShardRange.of( - new Tuple(TIMESTAMP_MILLIS, start), - new Tuple(TIMESTAMP_MILLIS, start + duration.toMillis())); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTuple.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTuple.java deleted file mode 100644 index 1fc358fb2297..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTuple.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThan; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class TestTuple -{ - @Test - public void testComparableTuple() - { - List types = ImmutableList.of(BIGINT, VARCHAR, BOOLEAN, DOUBLE, DATE, TIMESTAMP_MILLIS); - - Tuple tuple1 = new Tuple(types, ImmutableList.of(1L, "hello", false, 1.2d, 11111, 1112)); - Tuple equalToTuple1 = new Tuple(types, ImmutableList.of(1L, "hello", false, 1.2d, 11111, 1112)); - Tuple greaterThanTuple1 = new Tuple(types, ImmutableList.of(1L, "hello", false, 1.2d, 11111, 1113)); - Tuple lessThanTuple1 = new Tuple(types, ImmutableList.of(1L, "hello", false, 1.2d, 11111, 1111)); - - assertThat(tuple1.compareTo(equalToTuple1)).isEqualTo(0); - assertLessThan(tuple1.compareTo(greaterThanTuple1), 0); - assertGreaterThan(tuple1.compareTo(lessThanTuple1), 0); - } - - @Test - public void testMismatchedTypes() - { - assertThatThrownBy(() -> { - List types1 = ImmutableList.of(createVarcharType(3)); - List types2 = ImmutableList.of(createVarcharType(4)); - Tuple tuple1 = new Tuple(types1, ImmutableList.of("abc")); - Tuple tuple2 = new Tuple(types2, ImmutableList.of("abcd")); - tuple1.compareTo(tuple2); - }) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("types must be the same"); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/systemtables/TestShardMetadataRecordCursor.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/systemtables/TestShardMetadataRecordCursor.java deleted file mode 100644 index d00110f58d4f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/systemtables/TestShardMetadataRecordCursor.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.RaptorMetadata; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.predicate.ValueSet; -import io.trino.spi.type.Type; -import io.trino.testing.MaterializedRow; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.systemtables.ShardMetadataRecordCursor.SHARD_METADATA; -import static io.trino.spi.predicate.Range.greaterThan; -import static io.trino.spi.predicate.Range.lessThanOrEqual; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.MaterializedResult.DEFAULT_PRECISION; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardMetadataRecordCursor -{ - private static final SchemaTableName DEFAULT_TEST_ORDERS = new SchemaTableName("test", "orders"); - - private Handle dummyHandle; - private ConnectorMetadata metadata; - private Jdbi dbi; - - @BeforeEach - public void setup() - { - dbi = createTestingJdbi(); - this.dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - this.metadata = new RaptorMetadata(dbi, createShardManager(dbi)); - - // Create table - ConnectorTableMetadata table = tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("orderdate", DATE) - .property("temporal_column", "orderdate") - .build(); - createTable(table); - } - - @AfterEach - public void teardown() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testSimple() - { - ShardManager shardManager = createShardManager(dbi); - - // Add shards to the table - long tableId = 1; - OptionalInt bucketNumber = OptionalInt.empty(); - UUID uuid1 = UUID.randomUUID(); - UUID uuid2 = UUID.randomUUID(); - UUID uuid3 = UUID.randomUUID(); - ShardInfo shardInfo1 = new ShardInfo(uuid1, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 1, 10, 100, 0x1234); - ShardInfo shardInfo2 = new ShardInfo(uuid2, bucketNumber, ImmutableSet.of("node2"), ImmutableList.of(), 2, 20, 200, 0xCAFEBABEDEADBEEFL); - ShardInfo shardInfo3 = new ShardInfo(uuid3, bucketNumber, ImmutableSet.of("node3"), ImmutableList.of(), 3, 30, 300, 0xFEDCBA0987654321L); - List shards = ImmutableList.of(shardInfo1, shardInfo2, shardInfo3); - - long transactionId = shardManager.beginTransaction(); - - shardManager.commitShards( - transactionId, - tableId, - ImmutableList.of( - new ColumnInfo(1, BIGINT), - new ColumnInfo(2, DATE)), - shards, - Optional.empty(), - 0); - - Slice schema = utf8Slice(DEFAULT_TEST_ORDERS.getSchemaName()); - Slice table = utf8Slice(DEFAULT_TEST_ORDERS.getTableName()); - - DateTime date1 = DateTime.parse("2015-01-01T00:00"); - DateTime date2 = DateTime.parse("2015-01-02T00:00"); - TupleDomain tupleDomain = TupleDomain.withColumnDomains( - ImmutableMap.builder() - .put(0, Domain.singleValue(createVarcharType(10), schema)) - .put(1, Domain.create(ValueSet.ofRanges(lessThanOrEqual(createVarcharType(10), table)), true)) - .put(8, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)) - .put(9, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)) - .buildOrThrow()); - - List actual; - try (RecordCursor cursor = new ShardMetadataSystemTable(dbi).cursor(null, SESSION, tupleDomain)) { - actual = getMaterializedResults(cursor, SHARD_METADATA.getColumns()); - } - assertThat(actual.size()).isEqualTo(3); - - List expected = ImmutableList.of( - new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid1.toString()), null, 100L, 10L, 1L, utf8Slice("0000000000001234"), null, null, null, null), - new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid2.toString()), null, 200L, 20L, 2L, utf8Slice("cafebabedeadbeef"), null, null, null, null), - new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid3.toString()), null, 300L, 30L, 3L, utf8Slice("fedcba0987654321"), null, null, null, null)); - - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testNoSchemaFilter() - { - // Create "orders" table in a different schema - createTable(tableMetadataBuilder(new SchemaTableName("other", "orders")) - .column("orderkey", BIGINT) - .build()); - - // Create another table that should not be selected - createTable(tableMetadataBuilder(new SchemaTableName("schema1", "foo")) - .column("orderkey", BIGINT) - .build()); - - TupleDomain tupleDomain = TupleDomain.withColumnDomains( - ImmutableMap.of(1, Domain.singleValue(createVarcharType(10), utf8Slice("orders")))); - - MetadataDao metadataDao = dummyHandle.attach(MetadataDao.class); - Set actual = ImmutableSet.copyOf(ShardMetadataRecordCursor.getTableIds(dbi, tupleDomain)); - Set expected = ImmutableSet.of( - metadataDao.getTableInformation("other", "orders").getTableId(), - metadataDao.getTableInformation("test", "orders").getTableId()); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testNoTableFilter() - { - // Create "orders" table in a different schema - createTable(tableMetadataBuilder(new SchemaTableName("test", "orders2")) - .column("orderkey", BIGINT) - .build()); - - // Create another table that should not be selected - createTable(tableMetadataBuilder(new SchemaTableName("schema1", "foo")) - .column("orderkey", BIGINT) - .build()); - - TupleDomain tupleDomain = TupleDomain.withColumnDomains( - ImmutableMap.of(0, Domain.singleValue(createVarcharType(10), utf8Slice("test")))); - - MetadataDao metadataDao = dummyHandle.attach(MetadataDao.class); - Set actual = ImmutableSet.copyOf(ShardMetadataRecordCursor.getTableIds(dbi, tupleDomain)); - Set expected = ImmutableSet.of( - metadataDao.getTableInformation("test", "orders").getTableId(), - metadataDao.getTableInformation("test", "orders2").getTableId()); - assertThat(actual).isEqualTo(expected); - } - - private void createTable(ConnectorTableMetadata table) - { - metadata.createTable(SESSION, table, SaveMode.FAIL); - } - - private static List getMaterializedResults(RecordCursor cursor, List columns) - { - List types = columns.stream().map(ColumnMetadata::getType).collect(toList()); - - ImmutableList.Builder rowBuilder = ImmutableList.builder(); - for (int i = 0; i < types.size(); i++) { - assertThat(cursor.getType(i)).isEqualTo(types.get(i)); - } - - while (cursor.advanceNextPosition()) { - List values = new ArrayList<>(); - for (int i = 0; i < columns.size(); i++) { - Type type = columns.get(i).getType(); - Class javaType = type.getJavaType(); - if (cursor.isNull(i)) { - values.add(null); - } - else if (javaType == boolean.class) { - values.add(cursor.getBoolean(i)); - } - else if (javaType == long.class) { - values.add(cursor.getLong(i)); - } - else if (javaType == double.class) { - values.add(cursor.getDouble(i)); - } - else if (javaType == Slice.class) { - values.add(cursor.getSlice(i)); - } - } - rowBuilder.add(new MaterializedRow(DEFAULT_PRECISION, values)); - } - return rowBuilder.build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/util/TestPrioritizedFifoExecutor.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/util/TestPrioritizedFifoExecutor.java deleted file mode 100644 index 300764f06b48..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/util/TestPrioritizedFifoExecutor.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestPrioritizedFifoExecutor -{ - private static final Comparator DUMMY_COMPARATOR = (unused, unused2) -> 0; - - private ExecutorService executor; - - @BeforeEach - public void setUp() - { - executor = newCachedThreadPool(); - } - - @AfterEach - public void tearDown() - { - executor.shutdownNow(); - } - - @Test - public void testCounter() - throws Exception - { - // enforce single thread - PrioritizedFifoExecutor executor = new PrioritizedFifoExecutor<>(this.executor, 1, DUMMY_COMPARATOR); - - int totalTasks = 100_000; - AtomicInteger counter = new AtomicInteger(); - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch completeLatch = new CountDownLatch(totalTasks); - List> futures = new ArrayList<>(); - - for (int i = 0; i < totalTasks; i++) { - int taskNumber = i; - futures.add(executor.submit(() -> { - try { - // wait for the go signal - - assertThat(awaitUninterruptibly(startLatch, 1, TimeUnit.MINUTES)).isTrue(); - assertThat(futures.get(taskNumber).isDone()).isFalse(); - - // intentional distinct read and write calls - int initialCount = counter.get(); - counter.set(initialCount + 1); - } - finally { - completeLatch.countDown(); - } - })); - } - - for (Future future : futures) { - assertThat(future.isDone()).isFalse(); - } - - // signal go and wait for tasks to complete - startLatch.countDown(); - assertThat(awaitUninterruptibly(completeLatch, 1, TimeUnit.MINUTES)).isTrue(); - - assertThat(counter.get()).isEqualTo(totalTasks); - // since this is a fifo executor with one thread and completeLatch is decremented inside the future, - // the last future may not be done yet, but all the rest must be - futures.getLast().get(1, TimeUnit.MINUTES); - for (Future future : futures) { - assertThat(future.isDone()).isTrue(); - } - } - - @Test - public void testSingleThreadBound() - { - testBound(1, 100_000); - } - - @Test - public void testDoubleThreadBound() - { - testBound(2, 100_000); - } - - private void testBound(int maxThreads, int totalTasks) - { - PrioritizedFifoExecutor boundedExecutor = new PrioritizedFifoExecutor<>(executor, maxThreads, DUMMY_COMPARATOR); - - AtomicInteger activeThreadCount = new AtomicInteger(); - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch completeLatch = new CountDownLatch(totalTasks); - AtomicBoolean failed = new AtomicBoolean(); - - for (int i = 0; i < totalTasks; i++) { - boundedExecutor.submit(() -> { - try { - // wait for the go signal - awaitUninterruptibly(startLatch); - - int count = activeThreadCount.incrementAndGet(); - if (count < 1 || count > maxThreads) { - failed.set(true); - } - activeThreadCount.decrementAndGet(); - } - finally { - completeLatch.countDown(); - } - }); - } - - // signal go and wait for tasks to complete - startLatch.countDown(); - assertThat(awaitUninterruptibly(completeLatch, 1, TimeUnit.MINUTES)).isTrue(); - - assertThat(failed.get()).isFalse(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/resources/io/trino/plugin/raptor/legacy/security/security.json b/plugin/trino-raptor-legacy/src/test/resources/io/trino/plugin/raptor/legacy/security/security.json deleted file mode 100644 index dbea06e30017..000000000000 --- a/plugin/trino-raptor-legacy/src/test/resources/io/trino/plugin/raptor/legacy/security/security.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "tables": [ - { - "user": "user", - "privileges": [ - "SELECT", - "OWNERSHIP" - ] - } - ], - "schemas": [ - { - "owner": false - } - ] -} - diff --git a/pom.xml b/pom.xml index d99015685d89..651b783926ba 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,6 @@ plugin/trino-pinot plugin/trino-postgresql plugin/trino-prometheus - plugin/trino-raptor-legacy plugin/trino-redis plugin/trino-redshift plugin/trino-resource-group-managers @@ -1400,12 +1399,6 @@ ${project.version} - - io.trino - trino-raptor-legacy - ${project.version} - - io.trino trino-re2j diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java index ba3dd24027f1..0950b188a0c6 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java @@ -73,7 +73,6 @@ public void extendEnvironment(Environment.Builder builder) "pinot", "postgresql", "prometheus", - "raptor_legacy", "redis", "redshift", "snowflake", diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/raptor_legacy.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/raptor_legacy.properties deleted file mode 100644 index 5e04b760e2ed..000000000000 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/raptor_legacy.properties +++ /dev/null @@ -1,4 +0,0 @@ -connector.name=raptor_legacy -metadata.db.type=h2 -metadata.db.filename=mem:raptor;DB_CLOSE_DELAY=-1 -storage.data-directory=var/data diff --git a/testing/trino-server-dev/etc/catalog/raptor.properties b/testing/trino-server-dev/etc/catalog/raptor.properties deleted file mode 100644 index 08a7d4352a07..000000000000 --- a/testing/trino-server-dev/etc/catalog/raptor.properties +++ /dev/null @@ -1,13 +0,0 @@ -# -# WARNING -# ^^^^^^^ -# This configuration file is for development only and should NOT be used -# in production. For example configuration, see the Trino documentation. -# - -connector.name=raptor_legacy -metadata.db.type=h2 -metadata.db.filename=mem:raptor;DB_CLOSE_DELAY=-1 -#metadata.db.type=mysql -#metadata.db.connections.max=500 -storage.data-directory=var/data diff --git a/testing/trino-server-dev/etc/config.properties b/testing/trino-server-dev/etc/config.properties index 24c61d89a0d3..d9ddf55aa5f8 100644 --- a/testing/trino-server-dev/etc/config.properties +++ b/testing/trino-server-dev/etc/config.properties @@ -34,7 +34,6 @@ plugin.bundles=\ ../../plugin/trino-cassandra/pom.xml,\ ../../plugin/trino-memory/pom.xml,\ ../../plugin/trino-jmx/pom.xml,\ - ../../plugin/trino-raptor-legacy/pom.xml,\ ../../plugin/trino-hive/pom.xml,\ ../../plugin/trino-hudi/pom.xml,\ ../../plugin/trino-example-http/pom.xml,\