From 52d5b302dfe1252928e39cdb5b10ae3e9aafa836 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Tue, 1 Oct 2024 08:32:05 -0700 Subject: [PATCH] Remove Raptor legacy connector The connector has no known users and forces some complexities on the rest of the codebase. To allow this next clean up steps the connector is removed. In a contributor call, on slack, and in other conversations no concerns were expressed by anybody about the removal. --- .github/workflows/ci.yml | 2 - core/trino-server/src/main/provisio/trino.xml | 6 - docs/release-template.md | 2 - .../io/trino/orc/OutputStreamOrcDataSink.java | 9 - plugin/trino-raptor-legacy/pom.xml | 306 ----- .../plugin/raptor/legacy/NodeSupplier.java | 23 - .../raptor/legacy/RaptorBucketFunction.java | 99 -- .../legacy/RaptorBucketedUpdateFunction.java | 34 - .../legacy/RaptorBucketedUpdateHandle.java | 28 - .../raptor/legacy/RaptorColumnHandle.java | 158 --- .../plugin/raptor/legacy/RaptorConnector.java | 237 ---- .../raptor/legacy/RaptorConnectorFactory.java | 94 -- .../plugin/raptor/legacy/RaptorErrorCode.java | 52 - .../legacy/RaptorInsertTableHandle.java | 137 -- .../plugin/raptor/legacy/RaptorMergeSink.java | 133 -- .../raptor/legacy/RaptorMergeTableHandle.java | 49 - .../plugin/raptor/legacy/RaptorMetadata.java | 1033 --------------- .../raptor/legacy/RaptorMetadataFactory.java | 41 - .../plugin/raptor/legacy/RaptorModule.java | 78 -- .../RaptorNodePartitioningProvider.java | 89 -- .../legacy/RaptorOutputTableHandle.java | 154 --- .../plugin/raptor/legacy/RaptorPageSink.java | 326 ----- .../raptor/legacy/RaptorPageSinkProvider.java | 101 -- .../legacy/RaptorPageSourceProvider.java | 101 -- .../legacy/RaptorPartitioningHandle.java | 78 -- .../plugin/raptor/legacy/RaptorPlugin.java | 54 - .../legacy/RaptorSessionProperties.java | 124 -- .../plugin/raptor/legacy/RaptorSplit.java | 124 -- .../raptor/legacy/RaptorSplitManager.java | 248 ---- .../raptor/legacy/RaptorTableHandle.java | 171 --- .../raptor/legacy/RaptorTableProperties.java | 145 --- .../legacy/RaptorTransactionHandle.java | 74 -- .../RaptorUnbucketedUpdateFunction.java | 42 - .../legacy/RaptorUnbucketedUpdateHandle.java | 22 - .../raptor/legacy/backup/BackupConfig.java | 91 -- .../raptor/legacy/backup/BackupManager.java | 171 --- .../raptor/legacy/backup/BackupModule.java | 99 -- .../legacy/backup/BackupOperationStats.java | 91 -- .../raptor/legacy/backup/BackupService.java | 19 - .../legacy/backup/BackupServiceManager.java | 38 - .../raptor/legacy/backup/BackupStore.java | 52 - .../legacy/backup/FileBackupConfig.java | 39 - .../legacy/backup/FileBackupModule.java | 31 - .../raptor/legacy/backup/FileBackupStore.java | 131 -- .../raptor/legacy/backup/ForHttpBackup.java | 31 - .../legacy/backup/HttpBackupConfig.java | 39 - .../legacy/backup/HttpBackupModule.java | 48 - .../raptor/legacy/backup/HttpBackupStore.java | 243 ---- .../legacy/backup/ManagedBackupStore.java | 96 -- .../legacy/backup/TimeoutBackupStore.java | 112 -- .../legacy/metadata/AssignmentLimiter.java | 142 --- .../raptor/legacy/metadata/BucketNode.java | 68 - .../legacy/metadata/BucketReassigner.java | 78 -- .../raptor/legacy/metadata/BucketShards.java | 75 -- .../raptor/legacy/metadata/ColumnInfo.java | 52 - .../legacy/metadata/ColumnMetadataRow.java | 61 - .../raptor/legacy/metadata/ColumnStats.java | 69 - .../legacy/metadata/DatabaseConfig.java | 37 - .../metadata/DatabaseMetadataModule.java | 99 -- .../legacy/metadata/DatabaseShardManager.java | 933 -------------- .../metadata/DatabaseShardRecorder.java | 66 - .../raptor/legacy/metadata/Distribution.java | 106 -- .../raptor/legacy/metadata/ForMetadata.java | 31 - .../legacy/metadata/H2DatabaseConfig.java | 35 - .../raptor/legacy/metadata/H2ShardDao.java | 38 - .../raptor/legacy/metadata/IndexInserter.java | 167 --- .../legacy/metadata/JdbcDatabaseConfig.java | 35 - .../legacy/metadata/MetadataConfig.java | 70 -- .../raptor/legacy/metadata/MetadataDao.java | 308 ----- .../raptor/legacy/metadata/MySqlShardDao.java | 47 - .../raptor/legacy/metadata/NodeSize.java | 74 -- .../raptor/legacy/metadata/RaptorNode.java | 38 - .../raptor/legacy/metadata/SchemaDao.java | 163 --- .../raptor/legacy/metadata/SchemaDaoUtil.java | 72 -- .../metadata/SchemaTableNameMapper.java | 34 - .../raptor/legacy/metadata/ShardCleaner.java | 519 -------- .../legacy/metadata/ShardCleanerConfig.java | 157 --- .../raptor/legacy/metadata/ShardDao.java | 232 ---- .../raptor/legacy/metadata/ShardDelta.java | 60 - .../raptor/legacy/metadata/ShardInfo.java | 131 -- .../raptor/legacy/metadata/ShardIterator.java | 228 ---- .../raptor/legacy/metadata/ShardManager.java | 136 -- .../raptor/legacy/metadata/ShardMetadata.java | 228 ---- .../raptor/legacy/metadata/ShardNodes.java | 74 -- .../legacy/metadata/ShardPredicate.java | 220 ---- .../raptor/legacy/metadata/ShardRecorder.java | 21 - .../plugin/raptor/legacy/metadata/Table.java | 115 -- .../raptor/legacy/metadata/TableColumn.java | 150 --- .../raptor/legacy/metadata/TableMetadata.java | 82 -- .../legacy/metadata/TableMetadataRow.java | 86 -- .../raptor/legacy/metadata/TableStatsRow.java | 96 -- .../raptor/legacy/metadata/ViewResult.java | 59 - .../legacy/security/RaptorSecurity.java | 21 - .../legacy/security/RaptorSecurityConfig.java | 35 - .../legacy/security/RaptorSecurityModule.java | 45 - .../raptor/legacy/storage/BackupStats.java | 114 -- .../raptor/legacy/storage/BucketBalancer.java | 377 ------ .../legacy/storage/BucketBalancerConfig.java | 51 - .../legacy/storage/ColumnIndexStatsUtils.java | 54 - .../legacy/storage/FileStorageService.java | 219 ---- .../legacy/storage/OrcFileMetadata.java | 70 -- .../legacy/storage/OrcFileRewriter.java | 184 --- .../raptor/legacy/storage/OrcFileWriter.java | 203 --- .../legacy/storage/RaptorPageSource.java | 350 ------ .../legacy/storage/RaptorStorageManager.java | 668 ---------- .../raptor/legacy/storage/ShardEjector.java | 275 ---- .../legacy/storage/ShardRecoveryManager.java | 489 -------- .../legacy/storage/ShardRecoveryStats.java | 151 --- .../raptor/legacy/storage/ShardRewriter.java | 25 - .../raptor/legacy/storage/ShardStats.java | 267 ---- .../raptor/legacy/storage/StorageManager.java | 47 - .../legacy/storage/StorageManagerConfig.java | 387 ------ .../raptor/legacy/storage/StorageModule.java | 81 -- .../legacy/storage/StoragePageSink.java | 41 - .../raptor/legacy/storage/StorageService.java | 35 - .../organization/CompactionSetCreator.java | 108 -- .../storage/organization/JobFactory.java | 19 - .../storage/organization/OrganizationJob.java | 116 -- .../organization/OrganizationJobFactory.java | 46 - .../storage/organization/OrganizationSet.java | 83 -- .../organization/ShardCompactionManager.java | 231 ---- .../storage/organization/ShardCompactor.java | 406 ------ .../storage/organization/ShardIndexInfo.java | 126 -- .../ShardOrganizationManager.java | 292 ----- .../storage/organization/ShardOrganizer.java | 120 -- .../organization/ShardOrganizerDao.java | 44 - .../organization/ShardOrganizerUtil.java | 249 ---- .../storage/organization/ShardRange.java | 98 -- .../organization/TableOrganizationInfo.java | 42 - .../organization/TemporalFunction.java | 85 -- .../legacy/storage/organization/Tuple.java | 114 -- .../systemtables/ColumnRangesSystemTable.java | 187 --- .../legacy/systemtables/PageListBuilder.java | 60 - .../PreparedStatementBuilder.java | 252 ---- .../legacy/systemtables/ResultSetValues.java | 139 --- .../ShardMetadataRecordCursor.java | 375 ------ .../ShardMetadataSystemTable.java | 58 - .../TableMetadataSystemTable.java | 228 ---- .../systemtables/TableStatsSystemTable.java | 122 -- .../legacy/systemtables/ValueBuffer.java | 158 --- .../plugin/raptor/legacy/util/ArrayUtil.java | 50 - .../plugin/raptor/legacy/util/Closer.java | 52 - .../raptor/legacy/util/ConcatPageSource.java | 99 -- .../raptor/legacy/util/DaoSupplier.java | 44 - .../raptor/legacy/util/DatabaseUtil.java | 193 --- .../raptor/legacy/util/MetadataUtil.java | 42 - .../plugin/raptor/legacy/util/PageBuffer.java | 124 -- .../legacy/util/PrioritizedFifoExecutor.java | 146 --- .../util/SynchronizedResultIterator.java | 65 - .../legacy/util/SyncingFileOutputStream.java | 80 -- .../plugin/raptor/legacy/util/UuidUtil.java | 89 -- .../legacy/BaseRaptorConnectorTest.java | 1111 ----------------- .../plugin/raptor/legacy/DatabaseTesting.java | 36 - .../raptor/legacy/RaptorQueryRunner.java | 229 ---- .../legacy/TestRaptorBucketFunction.java | 91 -- .../TestRaptorBucketedConnectorTest.java | 68 - .../raptor/legacy/TestRaptorConnector.java | 290 ----- .../legacy/TestRaptorConnectorTest.java | 30 - .../legacy/TestRaptorMySqlConnectorTest.java | 84 -- .../raptor/legacy/TestRaptorPlugin.java | 56 - .../backup/AbstractTestBackupStore.java | 80 -- .../legacy/backup/TestBackupConfig.java | 58 - .../legacy/backup/TestBackupManager.java | 230 ---- .../legacy/backup/TestFileBackupConfig.java | 45 - .../legacy/backup/TestFileBackupStore.java | 59 - .../legacy/backup/TestHttpBackupConfig.java | 45 - .../legacy/backup/TestHttpBackupStore.java | 107 -- .../backup/TestingHttpBackupResource.java | 150 --- .../raptor/legacy/metadata/ShardNode.java | 71 -- .../metadata/TestAssignmentLimiter.java | 87 -- .../legacy/metadata/TestDatabaseConfig.java | 44 - .../metadata/TestDatabaseShardManager.java | 818 ------------ .../legacy/metadata/TestH2DatabaseConfig.java | 44 - .../legacy/metadata/TestMetadataConfig.java | 54 - .../legacy/metadata/TestMetadataDao.java | 98 -- .../legacy/metadata/TestRaptorMetadata.java | 826 ------------ .../metadata/TestRaptorSplitManager.java | 217 ---- .../legacy/metadata/TestShardCleaner.java | 464 ------- .../metadata/TestShardCleanerConfig.java | 71 -- .../raptor/legacy/metadata/TestShardDao.java | 288 ----- .../legacy/metadata/TestShardPredicate.java | 135 -- .../legacy/metadata/TestingShardDao.java | 56 - .../security/TestRaptorFileBasedSecurity.java | 83 -- .../security/TestRaptorReadOnlySecurity.java | 36 - .../security/TestRaptorSecurityConfig.java | 46 - .../legacy/storage/InMemoryShardRecorder.java | 62 - .../raptor/legacy/storage/OrcTestingUtil.java | 81 -- .../legacy/storage/TestBucketBalancer.java | 315 ----- .../storage/TestBucketBalancerConfig.java | 51 - .../storage/TestFileStorageService.java | 131 -- .../storage/TestMissingShardComparator.java | 56 - .../legacy/storage/TestOrcFileRewriter.java | 334 ----- .../storage/TestRaptorStorageManager.java | 692 ---------- .../legacy/storage/TestShardEjector.java | 239 ---- .../legacy/storage/TestShardRecovery.java | 271 ---- .../legacy/storage/TestShardWriter.java | 208 --- .../storage/TestStorageManagerConfig.java | 132 -- .../TestCompactionSetCreator.java | 283 ----- .../organization/TestShardCompactor.java | 342 ----- .../TestShardOrganizationManager.java | 226 ---- .../organization/TestShardOrganizer.java | 60 - .../organization/TestShardOrganizerUtil.java | 245 ---- .../storage/organization/TestShardRange.java | 117 -- .../organization/TestTemporalFunction.java | 96 -- .../storage/organization/TestTuple.java | 64 - .../TestShardMetadataRecordCursor.java | 246 ---- .../util/TestPrioritizedFifoExecutor.java | 152 --- .../raptor/legacy/security/security.json | 17 - pom.xml | 7 - .../EnvMultinodeAllConnectors.java | 1 - .../multinode-all/raptor_legacy.properties | 4 - .../etc/catalog/raptor.properties | 13 - .../trino-server-dev/etc/config.properties | 1 - 213 files changed, 30781 deletions(-) delete mode 100644 plugin/trino-raptor-legacy/pom.xml delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/NodeSupplier.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketFunction.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateFunction.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorColumnHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnector.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnectorFactory.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorErrorCode.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorInsertTableHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeSink.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeTableHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadata.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadataFactory.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorModule.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorNodePartitioningProvider.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorOutputTableHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSink.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSinkProvider.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSourceProvider.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPartitioningHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPlugin.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSessionProperties.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplit.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplitManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableProperties.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTransactionHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateFunction.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateHandle.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupModule.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupOperationStats.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupService.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupServiceManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupStore.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupModule.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupStore.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ForHttpBackup.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupModule.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupStore.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ManagedBackupStore.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/TimeoutBackupStore.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/AssignmentLimiter.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketNode.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketReassigner.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketShards.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnInfo.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnMetadataRow.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnStats.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseMetadataModule.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardRecorder.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Distribution.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ForMetadata.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2DatabaseConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2ShardDao.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/IndexInserter.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/JdbcDatabaseConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataDao.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MySqlShardDao.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/NodeSize.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/RaptorNode.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDao.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDaoUtil.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaTableNameMapper.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleaner.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleanerConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDao.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDelta.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardInfo.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardIterator.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardMetadata.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardNodes.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardPredicate.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardRecorder.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Table.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableColumn.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadata.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadataRow.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableStatsRow.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ViewResult.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurity.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityModule.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BackupStats.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancer.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancerConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ColumnIndexStatsUtils.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/FileStorageService.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileMetadata.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileRewriter.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileWriter.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardEjector.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryStats.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRewriter.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManagerConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StoragePageSink.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageService.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/CompactionSetCreator.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/JobFactory.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJob.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJobFactory.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationSet.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactionManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactor.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardIndexInfo.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizationManager.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizer.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerDao.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerUtil.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardRange.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TableOrganizationInfo.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TemporalFunction.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/Tuple.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ColumnRangesSystemTable.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PageListBuilder.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PreparedStatementBuilder.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ResultSetValues.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataRecordCursor.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataSystemTable.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableStatsSystemTable.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ValueBuffer.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ArrayUtil.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/Closer.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DaoSupplier.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DatabaseUtil.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/MetadataUtil.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PageBuffer.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PrioritizedFifoExecutor.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SynchronizedResultIterator.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SyncingFileOutputStream.java delete mode 100644 plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/UuidUtil.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/BaseRaptorConnectorTest.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/DatabaseTesting.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/RaptorQueryRunner.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketFunction.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketedConnectorTest.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnectorTest.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorMySqlConnectorTest.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorPlugin.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/AbstractTestBackupStore.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupManager.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupStore.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupStore.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestingHttpBackupResource.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/ShardNode.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestAssignmentLimiter.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseShardManager.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestH2DatabaseConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataDao.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorMetadata.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorSplitManager.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleaner.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleanerConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardDao.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardPredicate.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestingShardDao.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorFileBasedSecurity.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorReadOnlySecurity.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorSecurityConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/InMemoryShardRecorder.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/OrcTestingUtil.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancer.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancerConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestFileStorageService.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestMissingShardComparator.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcFileRewriter.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardEjector.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardWriter.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestStorageManagerConfig.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestCompactionSetCreator.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizationManager.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizer.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizerUtil.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardRange.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTemporalFunction.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTuple.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/systemtables/TestShardMetadataRecordCursor.java delete mode 100644 plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/util/TestPrioritizedFifoExecutor.java delete mode 100644 plugin/trino-raptor-legacy/src/test/resources/io/trino/plugin/raptor/legacy/security/security.json delete mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/raptor_legacy.properties delete mode 100644 testing/trino-server-dev/etc/catalog/raptor.properties diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2f73c46481b..f92c551677d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -370,7 +370,6 @@ jobs: !:trino-phoenix5, !:trino-pinot, !:trino-postgresql, - !:trino-raptor-legacy, !:trino-redis, !:trino-redshift, !:trino-resource-group-managers, @@ -476,7 +475,6 @@ jobs: - { modules: plugin/trino-phoenix5 } - { modules: plugin/trino-pinot } - { modules: plugin/trino-postgresql } - - { modules: plugin/trino-raptor-legacy } - { modules: plugin/trino-redis } - { modules: plugin/trino-redshift } - { modules: plugin/trino-redshift, profile: cloud-tests } diff --git a/core/trino-server/src/main/provisio/trino.xml b/core/trino-server/src/main/provisio/trino.xml index e27ac02f7f38..5fd5c6e2415d 100644 --- a/core/trino-server/src/main/provisio/trino.xml +++ b/core/trino-server/src/main/provisio/trino.xml @@ -279,12 +279,6 @@ - - - - - - diff --git a/docs/release-template.md b/docs/release-template.md index d41cf1df4f37..50df150ce128 100644 --- a/docs/release-template.md +++ b/docs/release-template.md @@ -70,8 +70,6 @@ ## Prometheus connector -## Raptor connector - ## Redis connector ## Redshift connector diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OutputStreamOrcDataSink.java b/lib/trino-orc/src/main/java/io/trino/orc/OutputStreamOrcDataSink.java index c357bb330660..65925c8b56d4 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OutputStreamOrcDataSink.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OutputStreamOrcDataSink.java @@ -41,15 +41,6 @@ public static OutputStreamOrcDataSink create(TrinoOutputFile outputFile) return new OutputStreamOrcDataSink(outputFile.create(memoryContext), memoryContext); } - // Do not use this method, it is here only for io.trino.plugin.raptor.legacy.storage.OrcFileWriter.createOrcDataSink - // and it should be removed in the future - @Deprecated - public static OutputStreamOrcDataSink create(OutputStream outputStream) - throws IOException - { - return new OutputStreamOrcDataSink(outputStream, newSimpleAggregatedMemoryContext()); - } - private OutputStreamOrcDataSink(OutputStream outputStream, AggregatedMemoryContext memoryContext) { this.output = new OutputStreamSliceOutput(requireNonNull(outputStream, "outputStream is null")); diff --git a/plugin/trino-raptor-legacy/pom.xml b/plugin/trino-raptor-legacy/pom.xml deleted file mode 100644 index dde2b04ac386..000000000000 --- a/plugin/trino-raptor-legacy/pom.xml +++ /dev/null @@ -1,306 +0,0 @@ - - - 4.0.0 - - - io.trino - trino-root - 460-SNAPSHOT - ../../pom.xml - - - trino-raptor-legacy - trino-plugin - Trino - Raptor legacy connector - - - - com.google.errorprone - error_prone_annotations - true - - - - com.google.guava - guava - - - - com.google.inject - guice - - - - com.h2database - h2 - - - - com.mysql - mysql-connector-j - - - - io.airlift - bootstrap - - - - io.airlift - concurrent - - - - io.airlift - configuration - - - - io.airlift - http-client - - - - io.airlift - json - - - - io.airlift - log - - - - io.airlift - stats - - - - io.airlift - units - - - - io.trino - trino-cache - - - - io.trino - trino-memory-context - - - - io.trino - trino-orc - - - - io.trino - trino-plugin-toolkit - - - - it.unimi.dsi - fastutil - - - - jakarta.annotation - jakarta.annotation-api - - - - jakarta.validation - jakarta.validation-api - - - - joda-time - joda-time - - - - org.gaul - modernizer-maven-annotations - - - - org.jdbi - jdbi3-core - - - - org.jdbi - jdbi3-sqlobject - - - - org.weakref - jmxutils - - - - com.fasterxml.jackson.core - jackson-annotations - provided - - - - io.airlift - slice - provided - - - - io.opentelemetry - opentelemetry-api - provided - - - - io.opentelemetry - opentelemetry-context - provided - - - - io.trino - trino-spi - provided - - - - org.openjdk.jol - jol-core - provided - - - - io.airlift - log-manager - runtime - - - - io.airlift - node - runtime - - - - io.airlift - http-server - test - - - - io.airlift - jaxrs - test - - - - io.airlift - junit-extensions - test - - - - io.airlift - testing - test - - - - io.trino - trino-client - test - - - - io.trino - trino-main - test-jar - test - - - - io.trino - trino-main - test - - - - io.trino - trino-testing - test - - - - io.trino - trino-testing-services - test - - - - io.trino - trino-tpch - test - - - - io.trino.tpch - tpch - test - - - - jakarta.servlet - jakarta.servlet-api - test - - - - jakarta.ws.rs - jakarta.ws.rs-api - test - - - - org.assertj - assertj-core - test - - - - org.jetbrains - annotations - test - - - - org.junit.jupiter - junit-jupiter-api - test - - - - org.junit.jupiter - junit-jupiter-engine - test - - - - org.testcontainers - mysql - test - - - - org.testcontainers - testcontainers - test - - - diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/NodeSupplier.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/NodeSupplier.java deleted file mode 100644 index db05b7a04b16..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/NodeSupplier.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.spi.Node; - -import java.util.Set; - -public interface NodeSupplier -{ - Set getWorkerNodes(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketFunction.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketFunction.java deleted file mode 100644 index 5ed85dc5b892..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketFunction.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.airlift.slice.XxHash64; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.VarcharType.VARCHAR; - -public class RaptorBucketFunction - implements BucketFunction -{ - private final HashFunction[] functions; - private final int bucketCount; - - public RaptorBucketFunction(int bucketCount, List types) - { - checkArgument(bucketCount > 0, "bucketCount must be at least one"); - this.bucketCount = bucketCount; - this.functions = types.stream() - .map(RaptorBucketFunction::getHashFunction) - .toArray(HashFunction[]::new); - } - - @SuppressWarnings("NumericCastThatLosesPrecision") - @Override - public int getBucket(Page page, int position) - { - long hash = 0; - for (int i = 0; i < page.getChannelCount(); i++) { - Block block = page.getBlock(i); - long value = functions[i].hash(block, position); - hash = (hash * 31) + value; - } - int value = (int) (hash & Integer.MAX_VALUE); - return value % bucketCount; - } - - public static void validateBucketType(Type type) - { - getHashFunction(type); - } - - private static HashFunction getHashFunction(Type type) - { - if (type.equals(BIGINT)) { - return bigintHashFunction(); - } - if (type.equals(INTEGER)) { - return intHashFunction(); - } - if (type instanceof VarcharType) { - return varcharHashFunction(); - } - throw new TrinoException(NOT_SUPPORTED, "Bucketing is supported for bigint, integer and varchar, not " + type.getDisplayName()); - } - - private static HashFunction bigintHashFunction() - { - return (block, position) -> XxHash64.hash(BIGINT.getLong(block, position)); - } - - private static HashFunction intHashFunction() - { - return (block, position) -> XxHash64.hash(INTEGER.getInt(block, position)); - } - - private static HashFunction varcharHashFunction() - { - return (block, position) -> XxHash64.hash(VARCHAR.getSlice(block, position)); - } - - private interface HashFunction - { - long hash(Block block, int position); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateFunction.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateFunction.java deleted file mode 100644 index 0e50dcdb4c07..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateFunction.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.RowBlock; -import io.trino.spi.block.SqlRow; -import io.trino.spi.connector.BucketFunction; - -import static io.trino.spi.type.IntegerType.INTEGER; - -public class RaptorBucketedUpdateFunction - implements BucketFunction -{ - @Override - public int getBucket(Page page, int position) - { - Block block = page.getBlock(0); - SqlRow row = ((RowBlock) block.getUnderlyingValueBlock()).getRow(block.getUnderlyingValuePosition(position)); - return INTEGER.getInt(row.getRawFieldBlock(0), row.getRawIndex()); // bucket field of row ID - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateHandle.java deleted file mode 100644 index 2c0d7ff6a465..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorBucketedUpdateHandle.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; - -import java.util.List; - -public class RaptorBucketedUpdateHandle - extends RaptorPartitioningHandle -{ - @JsonCreator - public RaptorBucketedUpdateHandle(long distributionId, List bucketToNode) - { - super(distributionId, bucketToNode); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorColumnHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorColumnHandle.java deleted file mode 100644 index 7cf69e07665c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorColumnHandle.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.type.Type; - -import java.util.Objects; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RowType.field; -import static io.trino.spi.type.RowType.rowType; -import static io.trino.spi.type.UuidType.UUID; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static java.util.Objects.requireNonNull; - -public final class RaptorColumnHandle - implements ColumnHandle -{ - // Generated rowId column for updates - private static final long SHARD_ROW_ID_COLUMN_ID = -1; - - public static final long SHARD_UUID_COLUMN_ID = -2; - public static final String SHARD_UUID_COLUMN_NAME = "$shard_uuid"; - public static final Type SHARD_UUID_COLUMN_TYPE = createVarcharType(36); - - public static final long BUCKET_NUMBER_COLUMN_ID = -3; - public static final String BUCKET_NUMBER_COLUMN_NAME = "$bucket_number"; - - private static final long MERGE_ROW_ID_COLUMN_ID = -4; - private static final String MERGE_ROW_ID_COLUMN_NAME = "$merge_row_id"; - private static final Type MERGE_ROW_ID_COLUMN_TYPE = rowType( - field("bucket", INTEGER), - field("uuid", UUID), - field("row_id", BIGINT)); - - private final String columnName; - private final long columnId; - private final Type columnType; - - @JsonCreator - public RaptorColumnHandle( - @JsonProperty("columnName") String columnName, - @JsonProperty("columnId") long columnId, - @JsonProperty("columnType") Type columnType) - { - this.columnName = requireNonNull(columnName, "columnName is null"); - this.columnId = columnId; - this.columnType = requireNonNull(columnType, "columnType is null"); - } - - @JsonProperty - public String getColumnName() - { - return columnName; - } - - @JsonProperty - public long getColumnId() - { - return columnId; - } - - @JsonProperty - public Type getColumnType() - { - return columnType; - } - - @Override - public String toString() - { - return columnName + ":" + columnId + ":" + columnType; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - RaptorColumnHandle other = (RaptorColumnHandle) obj; - return this.columnId == other.columnId; - } - - @Override - public int hashCode() - { - return Objects.hash(columnId); - } - - public boolean isShardUuid() - { - return isShardUuidColumn(columnId); - } - - public boolean isBucketNumber() - { - return isBucketNumberColumn(columnId); - } - - public static boolean isShardRowIdColumn(long columnId) - { - return columnId == SHARD_ROW_ID_COLUMN_ID; - } - - public static boolean isShardUuidColumn(long columnId) - { - return columnId == SHARD_UUID_COLUMN_ID; - } - - public static RaptorColumnHandle shardUuidColumnHandle() - { - return new RaptorColumnHandle(SHARD_UUID_COLUMN_NAME, SHARD_UUID_COLUMN_ID, SHARD_UUID_COLUMN_TYPE); - } - - public static boolean isBucketNumberColumn(long columnId) - { - return columnId == BUCKET_NUMBER_COLUMN_ID; - } - - public static RaptorColumnHandle bucketNumberColumnHandle() - { - return new RaptorColumnHandle(BUCKET_NUMBER_COLUMN_NAME, BUCKET_NUMBER_COLUMN_ID, INTEGER); - } - - public static RaptorColumnHandle mergeRowIdHandle() - { - return new RaptorColumnHandle(MERGE_ROW_ID_COLUMN_NAME, MERGE_ROW_ID_COLUMN_ID, MERGE_ROW_ID_COLUMN_TYPE); - } - - public static boolean isMergeRowIdColumn(long columnId) - { - return columnId == MERGE_ROW_ID_COLUMN_ID; - } - - public static boolean isHiddenColumn(long columnId) - { - return columnId < 0; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnector.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnector.java deleted file mode 100644 index 8e362eca5fa0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnector.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.SetMultimap; -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.spi.NodeManager; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorAccessControl; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorNodePartitioningProvider; -import io.trino.spi.connector.ConnectorPageSinkProvider; -import io.trino.spi.connector.ConnectorPageSourceProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.session.PropertyMetadata; -import io.trino.spi.transaction.IsolationLevel; -import jakarta.annotation.PostConstruct; -import org.jdbi.v3.core.Jdbi; - -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ScheduledExecutorService; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Verify.verify; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; -import static io.trino.spi.transaction.IsolationLevel.checkConnectorSupports; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class RaptorConnector - implements Connector -{ - private static final Logger log = Logger.get(RaptorConnector.class); - - private final LifeCycleManager lifeCycleManager; - private final RaptorMetadataFactory metadataFactory; - private final RaptorSplitManager splitManager; - private final RaptorPageSourceProvider pageSourceProvider; - private final RaptorPageSinkProvider pageSinkProvider; - private final RaptorNodePartitioningProvider nodePartitioningProvider; - private final List> sessionProperties; - private final List> tableProperties; - private final Set systemTables; - private final MetadataDao dao; - private final Optional accessControl; - private final boolean coordinator; - - private final ConcurrentMap transactions = new ConcurrentHashMap<>(); - - private final ScheduledExecutorService unblockMaintenanceExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("raptor-unblock-maintenance")); - - @GuardedBy("this") - private final SetMultimap deletions = HashMultimap.create(); - - @Inject - public RaptorConnector( - LifeCycleManager lifeCycleManager, - NodeManager nodeManager, - RaptorMetadataFactory metadataFactory, - RaptorSplitManager splitManager, - RaptorPageSourceProvider pageSourceProvider, - RaptorPageSinkProvider pageSinkProvider, - RaptorNodePartitioningProvider nodePartitioningProvider, - RaptorSessionProperties sessionProperties, - RaptorTableProperties tableProperties, - Set systemTables, - Optional accessControl, - @ForMetadata Jdbi dbi) - { - this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); - this.metadataFactory = requireNonNull(metadataFactory, "metadataFactory is null"); - this.splitManager = requireNonNull(splitManager, "splitManager is null"); - this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); - this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null"); - this.nodePartitioningProvider = requireNonNull(nodePartitioningProvider, "nodePartitioningProvider is null"); - this.sessionProperties = sessionProperties.getSessionProperties(); - this.tableProperties = tableProperties.getTableProperties(); - this.systemTables = requireNonNull(systemTables, "systemTables is null"); - this.accessControl = requireNonNull(accessControl, "accessControl is null"); - this.dao = onDemandDao(dbi, MetadataDao.class); - this.coordinator = nodeManager.getCurrentNode().isCoordinator(); - } - - @PostConstruct - public void start() - { - if (coordinator) { - dao.unblockAllMaintenance(); - } - } - - @Override - public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommit) - { - checkConnectorSupports(READ_COMMITTED, isolationLevel); - RaptorTransactionHandle transaction = new RaptorTransactionHandle(); - transactions.put(transaction, metadataFactory.create(tableId -> beginDelete(tableId, transaction.getUuid()))); - return transaction; - } - - @Override - public void commit(ConnectorTransactionHandle transaction) - { - checkArgument(transactions.remove(transaction) != null, "no such transaction: %s", transaction); - finishDelete(((RaptorTransactionHandle) transaction).getUuid()); - } - - @Override - public void rollback(ConnectorTransactionHandle transaction) - { - RaptorMetadata metadata = transactions.remove(transaction); - checkArgument(metadata != null, "no such transaction: %s", transaction); - finishDelete(((RaptorTransactionHandle) transaction).getUuid()); - metadata.rollback(); - } - - @Override - public ConnectorPageSourceProvider getPageSourceProvider() - { - return pageSourceProvider; - } - - @Override - public ConnectorPageSinkProvider getPageSinkProvider() - { - return pageSinkProvider; - } - - @Override - public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transaction) - { - RaptorMetadata metadata = transactions.get(transaction); - checkArgument(metadata != null, "no such transaction: %s", transaction); - return metadata; - } - - @Override - public ConnectorSplitManager getSplitManager() - { - return splitManager; - } - - @Override - public ConnectorNodePartitioningProvider getNodePartitioningProvider() - { - return nodePartitioningProvider; - } - - @Override - public List> getSessionProperties() - { - return sessionProperties; - } - - @Override - public List> getTableProperties() - { - return tableProperties; - } - - @Override - public Set getSystemTables() - { - return systemTables; - } - - @Override - public ConnectorAccessControl getAccessControl() - { - return accessControl.orElseThrow(UnsupportedOperationException::new); - } - - @Override - public final void shutdown() - { - lifeCycleManager.stop(); - } - - private synchronized void beginDelete(long tableId, UUID transactionId) - { - dao.blockMaintenance(tableId); - verify(deletions.put(tableId, transactionId)); - } - - private synchronized void finishDelete(UUID transactionId) - { - deletions.entries().stream() - .filter(entry -> entry.getValue().equals(transactionId)) - .findFirst() - .ifPresent(entry -> { - long tableId = entry.getKey(); - deletions.remove(tableId, transactionId); - if (!deletions.containsKey(tableId)) { - unblockMaintenance(tableId); - } - }); - } - - private void unblockMaintenance(long tableId) - { - try { - dao.unblockMaintenance(tableId); - } - catch (Throwable t) { - log.warn(t, "Failed to unblock maintenance for table ID %s, will retry", tableId); - unblockMaintenanceExecutor.schedule(() -> unblockMaintenance(tableId), 2, SECONDS); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnectorFactory.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnectorFactory.java deleted file mode 100644 index be1399b59a4b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorConnectorFactory.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableMap; -import com.google.inject.Injector; -import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; -import io.trino.plugin.base.CatalogNameModule; -import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; -import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.plugin.raptor.legacy.backup.BackupModule; -import io.trino.plugin.raptor.legacy.security.RaptorSecurityModule; -import io.trino.plugin.raptor.legacy.storage.StorageModule; -import io.trino.spi.NodeManager; -import io.trino.spi.PageSorter; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; -import org.weakref.jmx.guice.MBeanModule; - -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static io.trino.plugin.base.Versions.checkStrictSpiVersionMatch; -import static java.util.Objects.requireNonNull; - -public class RaptorConnectorFactory - implements ConnectorFactory -{ - private final String name; - private final Module metadataModule; - private final Map backupProviders; - - public RaptorConnectorFactory(String name, Module metadataModule, Map backupProviders) - { - checkArgument(!isNullOrEmpty(name), "name is null or empty"); - this.name = name; - this.metadataModule = requireNonNull(metadataModule, "metadataModule is null"); - this.backupProviders = ImmutableMap.copyOf(requireNonNull(backupProviders, "backupProviders is null")); - } - - @Override - public String getName() - { - return name; - } - - @Override - public Connector create(String catalogName, Map config, ConnectorContext context) - { - checkStrictSpiVersionMatch(context, this); - - Bootstrap app = new Bootstrap( - new CatalogNameModule(catalogName), - new JsonModule(), - new MBeanModule(), - new ConnectorObjectNameGeneratorModule("io.trino.plugin.raptor.legacy", "trino.plugin.raptor.legacy"), - new MBeanServerModule(), - binder -> { - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(PageSorter.class).toInstance(context.getPageSorter()); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }, - metadataModule, - new BackupModule(backupProviders), - new StorageModule(), - new RaptorModule(), - new RaptorSecurityModule()); - - Injector injector = app - .doNotInitializeLogging() - .setRequiredConfigurationProperties(config) - .initialize(); - - return injector.getInstance(RaptorConnector.class); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorErrorCode.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorErrorCode.java deleted file mode 100644 index e16ba20ef687..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorErrorCode.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.spi.ErrorCode; -import io.trino.spi.ErrorCodeSupplier; -import io.trino.spi.ErrorType; - -import static io.trino.spi.ErrorType.EXTERNAL; - -public enum RaptorErrorCode - implements ErrorCodeSupplier -{ - RAPTOR_ERROR(0, EXTERNAL), - RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS(1, EXTERNAL), - RAPTOR_NO_HOST_FOR_SHARD(2, EXTERNAL), - RAPTOR_RECOVERY_ERROR(3, EXTERNAL), - RAPTOR_BACKUP_TIMEOUT(4, EXTERNAL), - RAPTOR_METADATA_ERROR(5, EXTERNAL), - RAPTOR_BACKUP_ERROR(6, EXTERNAL), - RAPTOR_BACKUP_NOT_FOUND(7, EXTERNAL), - RAPTOR_REASSIGNMENT_DELAY(8, EXTERNAL), - RAPTOR_REASSIGNMENT_THROTTLE(9, EXTERNAL), - RAPTOR_RECOVERY_TIMEOUT(10, EXTERNAL), - RAPTOR_CORRUPT_METADATA(11, EXTERNAL), - RAPTOR_LOCAL_DISK_FULL(12, EXTERNAL), - RAPTOR_BACKUP_CORRUPTION(13, EXTERNAL); - - private final ErrorCode errorCode; - - RaptorErrorCode(int code, ErrorType type) - { - errorCode = new ErrorCode(code + 0x0300_0000, name(), type); - } - - @Override - public ErrorCode toErrorCode() - { - return errorCode; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorInsertTableHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorInsertTableHandle.java deleted file mode 100644 index 551e7c4647f7..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorInsertTableHandle.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class RaptorInsertTableHandle - implements ConnectorInsertTableHandle -{ - private final long transactionId; - private final long tableId; - private final List columnHandles; - private final List columnTypes; - private final Optional externalBatchId; - private final List sortColumnHandles; - private final List sortOrders; - private final OptionalInt bucketCount; - private final List bucketColumnHandles; - private final Optional temporalColumnHandle; - - @JsonCreator - public RaptorInsertTableHandle( - @JsonProperty("transactionId") long transactionId, - @JsonProperty("tableId") long tableId, - @JsonProperty("columnHandles") List columnHandles, - @JsonProperty("columnTypes") List columnTypes, - @JsonProperty("externalBatchId") Optional externalBatchId, - @JsonProperty("sortColumnHandles") List sortColumnHandles, - @JsonProperty("sortOrders") List sortOrders, - @JsonProperty("bucketCount") OptionalInt bucketCount, - @JsonProperty("bucketColumnHandles") List bucketColumnHandles, - @JsonProperty("temporalColumnHandle") Optional temporalColumnHandle) - { - checkArgument(tableId > 0, "tableId must be greater than zero"); - - this.transactionId = transactionId; - this.tableId = tableId; - this.columnHandles = ImmutableList.copyOf(requireNonNull(columnHandles, "columnHandles is null")); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.externalBatchId = requireNonNull(externalBatchId, "externalBatchId is null"); - - this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); - this.sortColumnHandles = ImmutableList.copyOf(requireNonNull(sortColumnHandles, "sortColumnHandles is null")); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.bucketColumnHandles = ImmutableList.copyOf(requireNonNull(bucketColumnHandles, "bucketColumnHandles is null")); - this.temporalColumnHandle = requireNonNull(temporalColumnHandle, "temporalColumnHandle is null"); - } - - @JsonProperty - public long getTransactionId() - { - return transactionId; - } - - @JsonProperty - public long getTableId() - { - return tableId; - } - - @JsonProperty - public List getColumnHandles() - { - return columnHandles; - } - - @JsonProperty - public List getColumnTypes() - { - return columnTypes; - } - - @JsonProperty - public Optional getExternalBatchId() - { - return externalBatchId; - } - - @JsonProperty - public List getSortColumnHandles() - { - return sortColumnHandles; - } - - @JsonProperty - public List getSortOrders() - { - return sortOrders; - } - - @JsonProperty - public OptionalInt getBucketCount() - { - return bucketCount; - } - - @JsonProperty - public List getBucketColumnHandles() - { - return bucketColumnHandles; - } - - @JsonProperty - public Optional getTemporalColumnHandle() - { - return temporalColumnHandle; - } - - @Override - public String toString() - { - return String.valueOf(tableId); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeSink.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeSink.java deleted file mode 100644 index 9f1ce30ae412..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeSink.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.trino.plugin.raptor.legacy.metadata.ShardDelta; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.ShardRewriter; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ConnectorMergeSink; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.MergePage; -import io.trino.spi.type.UuidType; - -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.OptionalInt; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; - -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.trino.spi.block.RowBlock.getRowFieldsFromBlock; -import static io.trino.spi.connector.MergePage.createDeleteAndInsertPages; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.UuidType.trinoUuidToJavaUuid; -import static java.lang.Math.toIntExact; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.allOf; -import static java.util.stream.Collectors.toUnmodifiableList; - -public class RaptorMergeSink - implements ConnectorMergeSink -{ - private static final JsonCodec SHARD_INFO_CODEC = jsonCodec(ShardInfo.class); - private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); - - private final ConnectorPageSink pageSink; - private final StorageManager storageManager; - private final long transactionId; - private final int columnCount; - private final Map> rowsToDelete = new HashMap<>(); - - public RaptorMergeSink(ConnectorPageSink pageSink, StorageManager storageManager, long transactionId, int columnCount) - { - this.pageSink = requireNonNull(pageSink, "pageSink is null"); - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - this.transactionId = transactionId; - this.columnCount = columnCount; - } - - @Override - public void storeMergedRows(Page page) - { - MergePage mergePage = createDeleteAndInsertPages(page, columnCount); - - mergePage.getInsertionsPage().ifPresent(pageSink::appendPage); - - mergePage.getDeletionsPage().ifPresent(deletions -> { - List fields = getRowFieldsFromBlock(deletions.getBlock(deletions.getChannelCount() - 1)); - Block shardBucketBlock = fields.get(0); - Block shardUuidBlock = fields.get(1); - Block shardRowIdBlock = fields.get(2); - - for (int position = 0; position < shardRowIdBlock.getPositionCount(); position++) { - OptionalInt bucketNumber = shardBucketBlock.isNull(position) - ? OptionalInt.empty() - : OptionalInt.of(INTEGER.getInt(shardBucketBlock, position)); - UUID uuid = trinoUuidToJavaUuid(UuidType.UUID.getSlice(shardUuidBlock, position)); - int rowId = toIntExact(BIGINT.getLong(shardRowIdBlock, position)); - Entry entry = rowsToDelete.computeIfAbsent(uuid, _ -> Map.entry(bucketNumber, new BitSet())); - verify(entry.getKey().equals(bucketNumber), "multiple bucket numbers for same shard"); - entry.getValue().set(rowId); - } - }); - } - - @Override - public CompletableFuture> finish() - { - List>> futures = new ArrayList<>(); - - rowsToDelete.forEach((uuid, entry) -> { - OptionalInt bucketNumber = entry.getKey(); - BitSet rowIds = entry.getValue(); - ShardRewriter rewriter = storageManager.createShardRewriter(transactionId, bucketNumber, uuid); - futures.add(rewriter.rewrite(rowIds)); - }); - - futures.add(pageSink.finish().thenApply(slices -> { - List newShards = slices.stream() - .map(slice -> SHARD_INFO_CODEC.fromJson(slice.getBytes())) - .collect(toImmutableList()); - ShardDelta delta = new ShardDelta(ImmutableList.of(), newShards); - return ImmutableList.of(Slices.wrappedBuffer(SHARD_DELTA_CODEC.toJsonBytes(delta))); - })); - - return allOf(futures.toArray(CompletableFuture[]::new)) - .thenApply(_ -> futures.stream() - .map(CompletableFuture::join) - .flatMap(Collection::stream) - .collect(toUnmodifiableList())); - } - - @Override - public void abort() - { - pageSink.abort(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeTableHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeTableHandle.java deleted file mode 100644 index 4240bd5c0e67..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMergeTableHandle.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.connector.ConnectorMergeTableHandle; - -import static java.util.Objects.requireNonNull; - -public class RaptorMergeTableHandle - implements ConnectorMergeTableHandle -{ - private final RaptorTableHandle tableHandle; - private final RaptorInsertTableHandle insertTableHandle; - - @JsonCreator - public RaptorMergeTableHandle( - @JsonProperty RaptorTableHandle tableHandle, - @JsonProperty RaptorInsertTableHandle insertTableHandle) - { - this.tableHandle = requireNonNull(tableHandle, "tableHandle is null"); - this.insertTableHandle = requireNonNull(insertTableHandle, "insertTableHandle is null"); - } - - @Override - @JsonProperty - public RaptorTableHandle getTableHandle() - { - return tableHandle; - } - - @JsonProperty - public RaptorInsertTableHandle getInsertTableHandle() - { - return insertTableHandle; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadata.java deleted file mode 100644 index 31fecf590e72..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadata.java +++ /dev/null @@ -1,1033 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimaps; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; -import io.airlift.log.Logger; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardDelta; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.plugin.raptor.legacy.metadata.ViewResult; -import io.trino.plugin.raptor.legacy.systemtables.ColumnRangesSystemTable; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorMergeTableHandle; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorOutputMetadata; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorPartitioningHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableLayout; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTablePartitioning; -import io.trino.spi.connector.ConnectorTableProperties; -import io.trino.spi.connector.ConnectorTableVersion; -import io.trino.spi.connector.ConnectorViewDefinition; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.ConstraintApplicationResult; -import io.trino.spi.connector.RetryMode; -import io.trino.spi.connector.RowChangeParadigm; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.connector.TableNotFoundException; -import io.trino.spi.connector.ViewNotFoundException; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.statistics.ComputedStatistics; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Jdbi; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.LongConsumer; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.collect.MoreCollectors.toOptional; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.trino.plugin.raptor.legacy.RaptorBucketFunction.validateBucketType; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.BUCKET_NUMBER_COLUMN_NAME; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_NAME; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.bucketNumberColumnHandle; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isHiddenColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.mergeRowIdHandle; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.shardUuidColumnHandle; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getExternalBatchId; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getOneSplitPerBucketThreshold; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.BUCKETED_ON_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.BUCKET_COUNT_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.DISTRIBUTION_NAME_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.ORDERING_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.ORGANIZED_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getBucketColumns; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getBucketCount; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getDistributionName; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getSortColumns; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.getTemporalColumn; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.isOrganized; -import static io.trino.plugin.raptor.legacy.systemtables.ColumnRangesSystemTable.getSourceTable; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.daoTransaction; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runIgnoringConstraintViolation; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runTransaction; -import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; -import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; -import static io.trino.spi.StandardErrorCode.NOT_FOUND; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.connector.RetryMode.NO_RETRIES; -import static io.trino.spi.connector.RowChangeParadigm.DELETE_ROW_AND_INSERT_ROW; -import static io.trino.spi.connector.SaveMode.REPLACE; -import static io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static java.lang.String.format; -import static java.util.Collections.nCopies; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toCollection; -import static java.util.stream.Collectors.toList; - -public class RaptorMetadata - implements ConnectorMetadata -{ - private static final Logger log = Logger.get(RaptorMetadata.class); - - private static final JsonCodec SHARD_INFO_CODEC = jsonCodec(ShardInfo.class); - private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); - - private static final JsonCodec VIEW_CODEC = - new JsonCodecFactory(new ObjectMapperProvider()).jsonCodec(ConnectorViewDefinition.class); - - private final Jdbi dbi; - private final MetadataDao dao; - private final ShardManager shardManager; - private final LongConsumer beginDeleteForTableId; - - private final AtomicReference currentTransactionId = new AtomicReference<>(); - - public RaptorMetadata(Jdbi dbi, ShardManager shardManager) - { - this(dbi, shardManager, tableId -> {}); - } - - public RaptorMetadata(Jdbi dbi, ShardManager shardManager, LongConsumer beginDeleteForTableId) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.dao = onDemandDao(dbi, MetadataDao.class); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.beginDeleteForTableId = requireNonNull(beginDeleteForTableId, "beginDeleteForTableId is null"); - } - - @Override - public List listSchemaNames(ConnectorSession session) - { - return dao.listSchemaNames(); - } - - @Override - public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName, Optional startVersion, Optional endVersion) - { - if (startVersion.isPresent() || endVersion.isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support versioned tables"); - } - - return getTableHandle(tableName); - } - - private RaptorTableHandle getTableHandle(SchemaTableName tableName) - { - requireNonNull(tableName, "tableName is null"); - Table table = dao.getTableInformation(tableName.getSchemaName(), tableName.getTableName()); - if (table == null) { - return null; - } - List tableColumns = dao.listTableColumns(table.getTableId()); - checkArgument(!tableColumns.isEmpty(), "Table '%s' does not have any columns", tableName); - - return new RaptorTableHandle( - tableName.getSchemaName(), - tableName.getTableName(), - table.getTableId(), - table.getDistributionId(), - table.getDistributionName(), - table.getBucketCount(), - table.isOrganized(), - TupleDomain.all(), - table.getDistributionId().map(shardManager::getBucketAssignments)); - } - - @Override - public Optional getSystemTable(ConnectorSession session, SchemaTableName tableName) - { - return getSourceTable(tableName) - .map(this::getTableHandle) - .map(handle -> new ColumnRangesSystemTable(handle, dbi)); - } - - @Override - public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) - { - RaptorTableHandle handle = (RaptorTableHandle) tableHandle; - SchemaTableName tableName = new SchemaTableName(handle.getSchemaName(), handle.getTableName()); - List tableColumns = dao.listTableColumns(handle.getTableId()); - if (tableColumns.isEmpty()) { - throw new TableNotFoundException(tableName); - } - - ImmutableMap.Builder properties = ImmutableMap.builder(); - SortedMap bucketing = new TreeMap<>(); - SortedMap ordering = new TreeMap<>(); - - for (TableColumn column : tableColumns) { - if (column.isTemporal()) { - properties.put(TEMPORAL_COLUMN_PROPERTY, column.getColumnName()); - } - column.getBucketOrdinal().ifPresent(bucketOrdinal -> bucketing.put(bucketOrdinal, column.getColumnName())); - column.getSortOrdinal().ifPresent(sortOrdinal -> ordering.put(sortOrdinal, column.getColumnName())); - } - - if (!bucketing.isEmpty()) { - properties.put(BUCKETED_ON_PROPERTY, ImmutableList.copyOf(bucketing.values())); - } - if (!ordering.isEmpty()) { - properties.put(ORDERING_PROPERTY, ImmutableList.copyOf(ordering.values())); - } - - handle.getBucketCount().ifPresent(bucketCount -> properties.put(BUCKET_COUNT_PROPERTY, bucketCount)); - handle.getDistributionName().ifPresent(distributionName -> properties.put(DISTRIBUTION_NAME_PROPERTY, distributionName)); - // Only display organization property if set - if (handle.isOrganized()) { - properties.put(ORGANIZED_PROPERTY, true); - } - - List columns = tableColumns.stream() - .map(TableColumn::toColumnMetadata) - .collect(toCollection(ArrayList::new)); - - columns.add(hiddenColumn(SHARD_UUID_COLUMN_NAME, SHARD_UUID_COLUMN_TYPE)); - - if (handle.isBucketed()) { - columns.add(hiddenColumn(BUCKET_NUMBER_COLUMN_NAME, INTEGER)); - } - - return new ConnectorTableMetadata(tableName, columns, properties.buildOrThrow()); - } - - @Override - public List listTables(ConnectorSession session, Optional schemaName) - { - // Deduplicate with set because state may change concurrently - return ImmutableSet.builder() - .addAll(dao.listTables(schemaName.orElse(null))) - .addAll(listViews(session, schemaName)) - .build().asList(); - } - - @Override - public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) - { - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (TableColumn tableColumn : dao.listTableColumns(raptorTableHandle.getTableId())) { - builder.put(tableColumn.getColumnName(), getRaptorColumnHandle(tableColumn)); - } - - RaptorColumnHandle uuidColumn = shardUuidColumnHandle(); - builder.put(uuidColumn.getColumnName(), uuidColumn); - - if (raptorTableHandle.isBucketed()) { - RaptorColumnHandle bucketNumberColumn = bucketNumberColumnHandle(); - builder.put(bucketNumberColumn.getColumnName(), bucketNumberColumn); - } - - return builder.buildOrThrow(); - } - - @Override - public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) - { - RaptorColumnHandle column = (RaptorColumnHandle) columnHandle; - - if (isHiddenColumn(column.getColumnId())) { - return hiddenColumn(column.getColumnName(), column.getColumnType()); - } - - return new ColumnMetadata(column.getColumnName(), column.getColumnType()); - } - - @Override - public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) - { - requireNonNull(prefix, "prefix is null"); - - ImmutableListMultimap.Builder columns = ImmutableListMultimap.builder(); - for (TableColumn tableColumn : dao.listTableColumns(prefix.getSchema().orElse(null), prefix.getTable().orElse(null))) { - ColumnMetadata columnMetadata = new ColumnMetadata(tableColumn.getColumnName(), tableColumn.getDataType()); - columns.put(tableColumn.getTable(), columnMetadata); - } - return Multimaps.asMap(columns.build()); - } - - @Override - public Optional> applyFilter(ConnectorSession session, ConnectorTableHandle handle, Constraint constraint) - { - RaptorTableHandle table = (RaptorTableHandle) handle; - TupleDomain newDomain = constraint.getSummary().transformKeys(RaptorColumnHandle.class::cast); - - if (newDomain.equals(table.getConstraint())) { - return Optional.empty(); - } - - return Optional.of(new ConstraintApplicationResult<>( - new RaptorTableHandle(table.getSchemaName(), - table.getTableName(), - table.getTableId(), - table.getDistributionId(), - table.getDistributionName(), - table.getBucketCount(), - table.isOrganized(), - newDomain.intersect(table.getConstraint()), - table.getBucketAssignments()), - constraint.getSummary(), - constraint.getExpression(), - false)); - } - - @Override - public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle handle) - { - RaptorTableHandle table = (RaptorTableHandle) handle; - - if (table.getPartitioningHandle().isEmpty()) { - return new ConnectorTableProperties(); - } - - List bucketColumnHandles = getBucketColumnHandles(table.getTableId()); - - RaptorPartitioningHandle partitioning = table.getPartitioningHandle().get(); - - boolean oneSplitPerBucket = table.getBucketCount().getAsInt() >= getOneSplitPerBucketThreshold(session); - - return new ConnectorTableProperties( - TupleDomain.all(), - Optional.of(new ConnectorTablePartitioning( - partitioning, - ImmutableList.copyOf(bucketColumnHandles), - oneSplitPerBucket)), - Optional.empty(), - ImmutableList.of()); - } - - @Override - public Optional getNewTableLayout(ConnectorSession session, ConnectorTableMetadata metadata) - { - ImmutableMap.Builder map = ImmutableMap.builder(); - long columnId = 1; - for (ColumnMetadata column : metadata.getColumns()) { - map.put(column.getName(), new RaptorColumnHandle(column.getName(), columnId, column.getType())); - columnId++; - } - - Optional distribution = getOrCreateDistribution(map.buildOrThrow(), metadata.getProperties()); - if (distribution.isEmpty()) { - return Optional.empty(); - } - - List partitionColumns = distribution.get().getBucketColumns().stream() - .map(RaptorColumnHandle::getColumnName) - .collect(toList()); - - long distributionId = distribution.get().getDistributionId(); - List bucketAssignments = shardManager.getBucketAssignments(distributionId); - ConnectorPartitioningHandle partitioning = new RaptorPartitioningHandle(distributionId, bucketAssignments); - - return Optional.of(new ConnectorTableLayout(partitioning, partitionColumns, false)); - } - - private Optional getOrCreateDistribution(Map columnHandleMap, Map properties) - { - OptionalInt bucketCount = getBucketCount(properties); - List bucketColumnHandles = getBucketColumnHandles(getBucketColumns(properties), columnHandleMap); - - if (bucketCount.isPresent() && bucketColumnHandles.isEmpty()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKETED_ON_PROPERTY, BUCKET_COUNT_PROPERTY)); - } - if (bucketCount.isEmpty() && !bucketColumnHandles.isEmpty()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKET_COUNT_PROPERTY, BUCKETED_ON_PROPERTY)); - } - ImmutableList.Builder bucketColumnTypes = ImmutableList.builder(); - for (RaptorColumnHandle column : bucketColumnHandles) { - validateBucketType(column.getColumnType()); - bucketColumnTypes.add(column.getColumnType()); - } - - long distributionId; - String distributionName = getDistributionName(properties); - if (distributionName != null) { - if (bucketColumnHandles.isEmpty()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKETED_ON_PROPERTY, DISTRIBUTION_NAME_PROPERTY)); - } - - Distribution distribution = dao.getDistribution(distributionName); - if (distribution == null) { - if (bucketCount.isEmpty()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Distribution does not exist and bucket count is not specified"); - } - distribution = getOrCreateDistribution(distributionName, bucketColumnTypes.build(), bucketCount.getAsInt()); - } - distributionId = distribution.getId(); - - if (bucketCount.isPresent() && (distribution.getBucketCount() != bucketCount.getAsInt())) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Bucket count must match distribution"); - } - if (!distribution.getColumnTypes().equals(bucketColumnTypes.build())) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Bucket column types must match distribution"); - } - } - else if (bucketCount.isPresent()) { - String types = Distribution.serializeColumnTypes(bucketColumnTypes.build()); - distributionId = dao.insertDistribution(null, types, bucketCount.getAsInt()); - } - else { - return Optional.empty(); - } - - shardManager.createBuckets(distributionId, bucketCount.getAsInt()); - - return Optional.of(new DistributionInfo(distributionId, bucketCount.getAsInt(), bucketColumnHandles)); - } - - private Distribution getOrCreateDistribution(String name, List columnTypes, int bucketCount) - { - String types = Distribution.serializeColumnTypes(columnTypes); - runIgnoringConstraintViolation(() -> dao.insertDistribution(name, types, bucketCount)); - - Distribution distribution = dao.getDistribution(name); - if (distribution == null) { - throw new TrinoException(RAPTOR_ERROR, "Distribution does not exist after insert"); - } - return distribution; - } - - @Override - public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, SaveMode saveMode) - { - if (saveMode == REPLACE) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support replacing tables"); - } - Optional layout = getNewTableLayout(session, tableMetadata); - finishCreateTable(session, beginCreateTable(session, tableMetadata, layout, NO_RETRIES, false), ImmutableList.of(), ImmutableList.of()); - } - - @Override - public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) - { - RaptorTableHandle raptorHandle = (RaptorTableHandle) tableHandle; - shardManager.dropTable(raptorHandle.getTableId()); - } - - @Override - public void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle, SchemaTableName newTableName) - { - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - runTransaction(dbi, handle -> { - MetadataDao dao = handle.attach(MetadataDao.class); - dao.renameTable(table.getTableId(), newTableName.getSchemaName(), newTableName.getTableName()); - return null; - }); - } - - @Override - public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) - { - if (column.getComment() != null) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support adding columns with comments"); - } - - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - - // Always add new columns to the end. - List existingColumns = dao.listTableColumns(table.getSchemaName(), table.getTableName()); - TableColumn lastColumn = existingColumns.getLast(); - long columnId = lastColumn.getColumnId() + 1; - int ordinalPosition = lastColumn.getOrdinalPosition() + 1; - - String type = column.getType().getTypeId().getId(); - daoTransaction(dbi, MetadataDao.class, dao -> { - dao.insertColumn(table.getTableId(), columnId, column.getName(), ordinalPosition, type, null, null); - dao.updateTableVersion(table.getTableId(), session.getStart().toEpochMilli()); - }); - - shardManager.addColumn(table.getTableId(), new ColumnInfo(columnId, column.getType())); - } - - @Override - public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) - { - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - RaptorColumnHandle sourceColumn = (RaptorColumnHandle) source; - daoTransaction(dbi, MetadataDao.class, dao -> { - dao.renameColumn(table.getTableId(), sourceColumn.getColumnId(), target); - dao.updateTableVersion(table.getTableId(), session.getStart().toEpochMilli()); - }); - } - - @Override - public void dropColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle column) - { - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - RaptorColumnHandle raptorColumn = (RaptorColumnHandle) column; - - List existingColumns = dao.listTableColumns(table.getSchemaName(), table.getTableName()); - if (existingColumns.size() <= 1) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop the only column in a table"); - } - long maxColumnId = existingColumns.stream().mapToLong(TableColumn::getColumnId).max().getAsLong(); - if (raptorColumn.getColumnId() == maxColumnId) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop the column which has the largest column ID in the table"); - } - - if (getBucketColumnHandles(table.getTableId()).contains(column)) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop bucket columns"); - } - - Optional.ofNullable(dao.getTemporalColumnId(table.getTableId())).ifPresent(tempColumnId -> { - if (raptorColumn.getColumnId() == tempColumnId) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop the temporal column"); - } - }); - - if (getSortColumnHandles(table.getTableId()).contains(raptorColumn)) { - throw new TrinoException(NOT_SUPPORTED, "Cannot drop sort columns"); - } - - daoTransaction(dbi, MetadataDao.class, dao -> { - dao.dropColumn(table.getTableId(), raptorColumn.getColumnId()); - dao.updateTableVersion(table.getTableId(), session.getStart().toEpochMilli()); - }); - - // TODO: drop column from index table - } - - @Override - public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout, RetryMode retryMode, boolean replace) - { - if (retryMode != NO_RETRIES) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support query retries"); - } - if (replace) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support replacing tables"); - } - if (tableMetadata.getComment().isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support creating tables with table comment"); - } - - if (viewExists(session, tableMetadata.getTable())) { - throw new TrinoException(ALREADY_EXISTS, "View already exists: " + tableMetadata.getTable()); - } - - Optional partitioning = layout - .map(ConnectorTableLayout::getPartitioning) - .map(Optional::get) - .map(RaptorPartitioningHandle.class::cast); - - ImmutableList.Builder columnHandles = ImmutableList.builder(); - ImmutableList.Builder columnTypes = ImmutableList.builder(); - - long columnId = 1; - for (ColumnMetadata column : tableMetadata.getColumns()) { - if (column.getComment() != null) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support creating tables with column comment"); - } - columnHandles.add(new RaptorColumnHandle(column.getName(), columnId, column.getType())); - columnTypes.add(column.getType()); - columnId++; - } - Map columnHandleMap = Maps.uniqueIndex(columnHandles.build(), RaptorColumnHandle::getColumnName); - - List sortColumnHandles = getSortColumnHandles(getSortColumns(tableMetadata.getProperties()), columnHandleMap); - Optional temporalColumnHandle = getTemporalColumnHandle(getTemporalColumn(tableMetadata.getProperties()), columnHandleMap); - - if (temporalColumnHandle.isPresent()) { - RaptorColumnHandle column = temporalColumnHandle.get(); - if (!column.getColumnType().equals(TIMESTAMP_MILLIS) && !column.getColumnType().equals(DATE)) { - throw new TrinoException(NOT_SUPPORTED, "Temporal column must be of type timestamp or date: " + column.getColumnName()); - } - } - - boolean organized = isOrganized(tableMetadata.getProperties()); - if (organized) { - if (temporalColumnHandle.isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "Table with temporal columns cannot be organized"); - } - if (sortColumnHandles.isEmpty()) { - throw new TrinoException(NOT_SUPPORTED, "Table organization requires an ordering"); - } - } - - long transactionId = shardManager.beginTransaction(); - - setTransactionId(transactionId); - - Optional distribution = partitioning.map(handle -> - getDistributionInfo(handle.getDistributionId(), columnHandleMap, tableMetadata.getProperties())); - - return new RaptorOutputTableHandle( - transactionId, - tableMetadata.getTable().getSchemaName(), - tableMetadata.getTable().getTableName(), - columnHandles.build(), - columnTypes.build(), - sortColumnHandles, - nCopies(sortColumnHandles.size(), ASC_NULLS_FIRST), - temporalColumnHandle, - distribution.map(info -> OptionalLong.of(info.getDistributionId())).orElse(OptionalLong.empty()), - distribution.map(info -> OptionalInt.of(info.getBucketCount())).orElse(OptionalInt.empty()), - organized, - distribution.map(DistributionInfo::getBucketColumns).orElse(ImmutableList.of())); - } - - private DistributionInfo getDistributionInfo(long distributionId, Map columnHandleMap, Map properties) - { - Distribution distribution = dao.getDistribution(distributionId); - if (distribution == null) { - throw new TrinoException(RAPTOR_ERROR, "Distribution ID does not exist: " + distributionId); - } - List bucketColumnHandles = getBucketColumnHandles(getBucketColumns(properties), columnHandleMap); - return new DistributionInfo(distributionId, distribution.getBucketCount(), bucketColumnHandles); - } - - private static Optional getTemporalColumnHandle(String temporalColumn, Map columnHandleMap) - { - if (temporalColumn == null) { - return Optional.empty(); - } - - RaptorColumnHandle handle = columnHandleMap.get(temporalColumn); - if (handle == null) { - throw new TrinoException(NOT_FOUND, "Temporal column does not exist: " + temporalColumn); - } - return Optional.of(handle); - } - - private static List getSortColumnHandles(List sortColumns, Map columnHandleMap) - { - ImmutableList.Builder columnHandles = ImmutableList.builder(); - for (String column : sortColumns) { - if (!columnHandleMap.containsKey(column)) { - throw new TrinoException(NOT_FOUND, "Ordering column does not exist: " + column); - } - columnHandles.add(columnHandleMap.get(column)); - } - return columnHandles.build(); - } - - private static List getBucketColumnHandles(List bucketColumns, Map columnHandleMap) - { - ImmutableList.Builder columnHandles = ImmutableList.builder(); - for (String column : bucketColumns) { - if (!columnHandleMap.containsKey(column)) { - throw new TrinoException(NOT_FOUND, "Bucketing column does not exist: " + column); - } - columnHandles.add(columnHandleMap.get(column)); - } - return columnHandles.build(); - } - - @Override - public Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle outputTableHandle, Collection fragments, Collection computedStatistics) - { - RaptorOutputTableHandle table = (RaptorOutputTableHandle) outputTableHandle; - long transactionId = table.getTransactionId(); - long updateTime = session.getStart().toEpochMilli(); - - long newTableId = runTransaction(dbi, dbiHandle -> { - MetadataDao dao = dbiHandle.attach(MetadataDao.class); - - Long distributionId = table.getDistributionId().isPresent() ? table.getDistributionId().getAsLong() : null; - // TODO: update default value of organization_enabled to true - long tableId = dao.insertTable(table.getSchemaName(), table.getTableName(), true, table.isOrganized(), distributionId, updateTime); - - List sortColumnHandles = table.getSortColumnHandles(); - List bucketColumnHandles = table.getBucketColumnHandles(); - - for (int i = 0; i < table.getColumnTypes().size(); i++) { - RaptorColumnHandle column = table.getColumnHandles().get(i); - - int columnId = i + 1; - String type = table.getColumnTypes().get(i).getTypeId().getId(); - Integer sortPosition = sortColumnHandles.contains(column) ? sortColumnHandles.indexOf(column) : null; - Integer bucketPosition = bucketColumnHandles.contains(column) ? bucketColumnHandles.indexOf(column) : null; - - dao.insertColumn(tableId, columnId, column.getColumnName(), i, type, sortPosition, bucketPosition); - - if (table.getTemporalColumnHandle().isPresent() && table.getTemporalColumnHandle().get().equals(column)) { - dao.updateTemporalColumnId(tableId, columnId); - } - } - - return tableId; - }); - - List columns = table.getColumnHandles().stream().map(ColumnInfo::fromHandle).collect(toList()); - - OptionalLong temporalColumnId = table.getTemporalColumnHandle().map(RaptorColumnHandle::getColumnId) - .map(OptionalLong::of) - .orElse(OptionalLong.empty()); - - // TODO: refactor this to avoid creating an empty table on failure - shardManager.createTable(newTableId, columns, table.getBucketCount().isPresent(), temporalColumnId); - shardManager.commitShards(transactionId, newTableId, columns, parseFragments(fragments), Optional.empty(), updateTime); - - clearRollback(); - - return Optional.empty(); - } - - @Override - public Optional getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) - { - RaptorTableHandle table = (RaptorTableHandle) tableHandle; - if (table.getPartitioningHandle().isEmpty()) { - return Optional.empty(); - } - return Optional.of(new ConnectorTableLayout( - table.getPartitioningHandle().get(), - getBucketColumnHandles(table.getTableId()).stream() - .map(RaptorColumnHandle::getColumnName) - .collect(toImmutableList()), - false)); - } - - @Override - public RaptorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle, List columns, RetryMode retryMode) - { - if (retryMode != NO_RETRIES) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support query retries"); - } - - RaptorTableHandle handle = (RaptorTableHandle) tableHandle; - long tableId = handle.getTableId(); - - ImmutableList.Builder columnHandlesBuilder = ImmutableList.builder(); - ImmutableList.Builder columnTypes = ImmutableList.builder(); - for (TableColumn column : dao.listTableColumns(tableId)) { - columnHandlesBuilder.add(new RaptorColumnHandle(column.getColumnName(), column.getColumnId(), column.getDataType())); - columnTypes.add(column.getDataType()); - } - - long transactionId = shardManager.beginTransaction(); - - setTransactionId(transactionId); - - Optional externalBatchId = getExternalBatchId(session); - List sortColumnHandles = getSortColumnHandles(tableId); - List bucketColumnHandles = getBucketColumnHandles(tableId); - - ImmutableList columnHandles = columnHandlesBuilder.build(); - Optional temporalColumnHandle = Optional.ofNullable(dao.getTemporalColumnId(tableId)) - .map(temporalColumnId -> getOnlyElement(columnHandles.stream() - .filter(columnHandle -> columnHandle.getColumnId() == temporalColumnId) - .collect(toList()))); - - return new RaptorInsertTableHandle( - transactionId, - tableId, - columnHandles, - columnTypes.build(), - externalBatchId, - sortColumnHandles, - nCopies(sortColumnHandles.size(), ASC_NULLS_FIRST), - handle.getBucketCount(), - bucketColumnHandles, - temporalColumnHandle); - } - - private List getSortColumnHandles(long tableId) - { - return dao.listSortColumns(tableId).stream() - .map(this::getRaptorColumnHandle) - .collect(toList()); - } - - private List getBucketColumnHandles(long tableId) - { - return dao.listBucketColumns(tableId).stream() - .map(this::getRaptorColumnHandle) - .collect(toList()); - } - - @Override - public Optional finishInsert( - ConnectorSession session, - ConnectorInsertTableHandle insertHandle, - List sourceTableHandles, - Collection fragments, - Collection computedStatistics) - { - RaptorInsertTableHandle handle = (RaptorInsertTableHandle) insertHandle; - long transactionId = handle.getTransactionId(); - long tableId = handle.getTableId(); - Optional externalBatchId = handle.getExternalBatchId(); - List columns = handle.getColumnHandles().stream().map(ColumnInfo::fromHandle).collect(toList()); - long updateTime = session.getStart().toEpochMilli(); - - Collection shards = parseFragments(fragments); - log.info("Committing insert into tableId %s (queryId: %s, shards: %s, columns: %s)", handle.getTableId(), session.getQueryId(), shards.size(), columns.size()); - shardManager.commitShards(transactionId, tableId, columns, shards, externalBatchId, updateTime); - - clearRollback(); - - return Optional.empty(); - } - - private void finishDelete(ConnectorSession session, RaptorTableHandle tableHandle, long transactionId, Collection fragments) - { - long tableId = tableHandle.getTableId(); - - List columns = getColumnHandles(session, tableHandle).values().stream() - .map(RaptorColumnHandle.class::cast) - .map(ColumnInfo::fromHandle).collect(toList()); - - Set oldShardUuids = new HashSet<>(); - List newShards = new ArrayList<>(); - - for (Slice fragment : fragments) { - ShardDelta delta = SHARD_DELTA_CODEC.fromJson(fragment.getBytes()); - for (UUID uuid : delta.getOldShardUuids()) { - verify(oldShardUuids.add(uuid), "duplicate old shard: %s", uuid); - } - newShards.addAll(delta.getNewShards()); - } - - OptionalLong updateTime = OptionalLong.of(session.getStart().toEpochMilli()); - - log.info("Finishing update for tableId %s (removed: %s, new: %s)", tableId, oldShardUuids.size(), newShards.size()); - shardManager.replaceShardUuids(transactionId, tableId, columns, oldShardUuids, newShards, updateTime); - - clearRollback(); - } - - @Override - public RowChangeParadigm getRowChangeParadigm(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return DELETE_ROW_AND_INSERT_ROW; - } - - @Override - public ColumnHandle getMergeRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return mergeRowIdHandle(); - } - - @Override - public Optional getUpdateLayout(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return ((RaptorTableHandle) tableHandle).getDistributionId().map(distributionId -> - new RaptorBucketedUpdateHandle(distributionId, shardManager.getBucketAssignments(distributionId))) - .or(() -> Optional.of(RaptorUnbucketedUpdateHandle.INSTANCE)); - } - - @Override - public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) - { - RaptorTableHandle handle = (RaptorTableHandle) tableHandle; - - beginDeleteForTableId.accept(handle.getTableId()); - - RaptorInsertTableHandle insertHandle = beginInsert(session, handle, ImmutableList.of(), retryMode); - - return new RaptorMergeTableHandle(handle, insertHandle); - } - - @Override - public void finishMerge(ConnectorSession session, ConnectorMergeTableHandle mergeTableHandle, List sourceTableHandles, Collection fragments, Collection computedStatistics) - { - RaptorMergeTableHandle handle = (RaptorMergeTableHandle) mergeTableHandle; - long transactionId = handle.getInsertTableHandle().getTransactionId(); - finishDelete(session, handle.getTableHandle(), transactionId, fragments); - } - - @Override - public void createView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition definition, Map viewProperties, boolean replace) - { - checkArgument(viewProperties.isEmpty(), "This connector does not support creating views with properties"); - String schemaName = viewName.getSchemaName(); - String tableName = viewName.getTableName(); - String viewData = VIEW_CODEC.toJson(definition); - - if (getTableHandle(viewName) != null) { - throw new TrinoException(ALREADY_EXISTS, "Table already exists: " + viewName); - } - - if (replace) { - daoTransaction(dbi, MetadataDao.class, dao -> { - dao.dropView(schemaName, tableName); - dao.insertView(schemaName, tableName, viewData); - }); - return; - } - - try { - dao.insertView(schemaName, tableName, viewData); - } - catch (TrinoException e) { - if (viewExists(session, viewName)) { - throw new TrinoException(ALREADY_EXISTS, "View already exists: " + viewName); - } - throw e; - } - } - - @Override - public void dropView(ConnectorSession session, SchemaTableName viewName) - { - if (!viewExists(session, viewName)) { - throw new ViewNotFoundException(viewName); - } - dao.dropView(viewName.getSchemaName(), viewName.getTableName()); - } - - @Override - public List listViews(ConnectorSession session, Optional schemaName) - { - return dao.listViews(schemaName.orElse(null)); - } - - @Override - public Map getViews(ConnectorSession session, Optional schemaName) - { - ImmutableMap.Builder map = ImmutableMap.builder(); - for (ViewResult view : dao.getViews(schemaName.orElse(null), null)) { - map.put(view.getName(), VIEW_CODEC.fromJson(view.getData())); - } - return map.buildOrThrow(); - } - - @Override - public Optional getView(ConnectorSession session, SchemaTableName viewName) - { - return dao.getViews(viewName.getSchemaName(), viewName.getTableName()).stream() - .map(view -> VIEW_CODEC.fromJson(view.getData())) - .collect(toOptional()); - } - - private boolean viewExists(ConnectorSession session, SchemaTableName viewName) - { - return getView(session, viewName).isPresent(); - } - - private RaptorColumnHandle getRaptorColumnHandle(TableColumn tableColumn) - { - return new RaptorColumnHandle(tableColumn.getColumnName(), tableColumn.getColumnId(), tableColumn.getDataType()); - } - - private static Collection parseFragments(Collection fragments) - { - return fragments.stream() - .map(fragment -> SHARD_INFO_CODEC.fromJson(fragment.getBytes())) - .collect(toList()); - } - - private static ColumnMetadata hiddenColumn(String name, Type type) - { - return ColumnMetadata.builder() - .setName(name) - .setType(type) - .setHidden(true) - .build(); - } - - private void setTransactionId(long transactionId) - { - checkState(currentTransactionId.compareAndSet(null, transactionId), "current transaction ID already set"); - } - - private void clearRollback() - { - currentTransactionId.set(null); - } - - public void rollback() - { - Long transactionId = currentTransactionId.getAndSet(null); - if (transactionId != null) { - shardManager.rollbackTransaction(transactionId); - } - } - - private static class DistributionInfo - { - private final long distributionId; - private final int bucketCount; - private final List bucketColumns; - - public DistributionInfo(long distributionId, int bucketCount, List bucketColumns) - { - this.distributionId = distributionId; - this.bucketCount = bucketCount; - this.bucketColumns = ImmutableList.copyOf(requireNonNull(bucketColumns, "bucketColumns is null")); - } - - public long getDistributionId() - { - return distributionId; - } - - public int getBucketCount() - { - return bucketCount; - } - - public List getBucketColumns() - { - return bucketColumns; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadataFactory.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadataFactory.java deleted file mode 100644 index 5a5f5865420e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorMetadataFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import org.jdbi.v3.core.Jdbi; - -import java.util.function.LongConsumer; - -import static java.util.Objects.requireNonNull; - -public class RaptorMetadataFactory -{ - private final Jdbi dbi; - private final ShardManager shardManager; - - @Inject - public RaptorMetadataFactory(@ForMetadata Jdbi dbi, ShardManager shardManager) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - } - - public RaptorMetadata create(LongConsumer beginDeleteForTableId) - { - return new RaptorMetadata(dbi, shardManager, beginDeleteForTableId); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorModule.java deleted file mode 100644 index 460cc13fd172..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorModule.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; -import com.google.inject.multibindings.Multibinder; -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.plugin.raptor.legacy.systemtables.ShardMetadataSystemTable; -import io.trino.plugin.raptor.legacy.systemtables.TableMetadataSystemTable; -import io.trino.plugin.raptor.legacy.systemtables.TableStatsSystemTable; -import io.trino.spi.NodeManager; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.type.TypeManager; -import org.jdbi.v3.core.ConnectionFactory; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.sqlobject.SqlObjectPlugin; - -import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; - -public class RaptorModule - implements Module -{ - @Override - public void configure(Binder binder) - { - binder.bind(RaptorConnector.class).in(Scopes.SINGLETON); - binder.bind(RaptorMetadataFactory.class).in(Scopes.SINGLETON); - binder.bind(RaptorSplitManager.class).in(Scopes.SINGLETON); - binder.bind(RaptorPageSourceProvider.class).in(Scopes.SINGLETON); - binder.bind(RaptorPageSinkProvider.class).in(Scopes.SINGLETON); - binder.bind(RaptorNodePartitioningProvider.class).in(Scopes.SINGLETON); - binder.bind(RaptorSessionProperties.class).in(Scopes.SINGLETON); - binder.bind(RaptorTableProperties.class).in(Scopes.SINGLETON); - - Multibinder tableBinder = newSetBinder(binder, SystemTable.class); - tableBinder.addBinding().to(ShardMetadataSystemTable.class).in(Scopes.SINGLETON); - tableBinder.addBinding().to(TableMetadataSystemTable.class).in(Scopes.SINGLETON); - tableBinder.addBinding().to(TableStatsSystemTable.class).in(Scopes.SINGLETON); - } - - @ForMetadata - @Singleton - @Provides - public static Jdbi createJdbi(@ForMetadata ConnectionFactory connectionFactory, TypeManager typeManager) - { - Jdbi dbi = Jdbi.create(connectionFactory) - .installPlugin(new SqlObjectPlugin()) - .registerRowMapper(new TableColumn.Mapper(typeManager)) - .registerRowMapper(new Distribution.Mapper(typeManager)); - createTablesWithRetry(dbi); - return dbi; - } - - @Provides - @Singleton - public static NodeSupplier createNodeSupplier(NodeManager nodeManager) - { - return nodeManager::getWorkerNodes; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorNodePartitioningProvider.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorNodePartitioningProvider.java deleted file mode 100644 index 7b8969efe494..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorNodePartitioningProvider.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.trino.spi.Node; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.connector.ConnectorBucketNodeMap; -import io.trino.spi.connector.ConnectorNodePartitioningProvider; -import io.trino.spi.connector.ConnectorPartitioningHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.ToIntFunction; - -import static com.google.common.collect.Maps.uniqueIndex; -import static io.trino.spi.StandardErrorCode.NO_NODES_AVAILABLE; -import static io.trino.spi.connector.ConnectorBucketNodeMap.createBucketNodeMap; -import static java.util.Objects.requireNonNull; - -public class RaptorNodePartitioningProvider - implements ConnectorNodePartitioningProvider -{ - private final NodeSupplier nodeSupplier; - - @Inject - public RaptorNodePartitioningProvider(NodeSupplier nodeSupplier) - { - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - } - - @Override - public Optional getBucketNodeMapping(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorPartitioningHandle partitioning) - { - if (partitioning instanceof RaptorUnbucketedUpdateHandle) { - return Optional.empty(); - } - - RaptorPartitioningHandle handle = (RaptorPartitioningHandle) partitioning; - - Map nodesById = uniqueIndex(nodeSupplier.getWorkerNodes(), Node::getNodeIdentifier); - - ImmutableList.Builder bucketToNode = ImmutableList.builder(); - for (String nodeIdentifier : handle.getBucketToNode()) { - Node node = nodesById.get(nodeIdentifier); - if (node == null) { - throw new TrinoException(NO_NODES_AVAILABLE, "Node for bucket is offline: " + nodeIdentifier); - } - bucketToNode.add(node); - } - return Optional.of(createBucketNodeMap(bucketToNode.build())); - } - - @Override - public ToIntFunction getSplitBucketFunction(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorPartitioningHandle partitioning) - { - return value -> ((RaptorSplit) value).getBucketNumber().getAsInt(); - } - - @Override - public BucketFunction getBucketFunction(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorPartitioningHandle partitioning, List partitionChannelTypes, int bucketCount) - { - if (partitioning instanceof RaptorUnbucketedUpdateHandle) { - return new RaptorUnbucketedUpdateFunction(bucketCount); - } - if (partitioning instanceof RaptorBucketedUpdateHandle) { - return new RaptorBucketedUpdateFunction(); - } - return new RaptorBucketFunction(bucketCount, partitionChannelTypes); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorOutputTableHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorOutputTableHandle.java deleted file mode 100644 index e512eb06fcee..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorOutputTableHandle.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static io.trino.plugin.raptor.legacy.util.MetadataUtil.checkSchemaName; -import static io.trino.plugin.raptor.legacy.util.MetadataUtil.checkTableName; -import static java.util.Objects.requireNonNull; - -public class RaptorOutputTableHandle - implements ConnectorOutputTableHandle -{ - private final long transactionId; - private final String schemaName; - private final String tableName; - private final List columnHandles; - private final List columnTypes; - private final List sortColumnHandles; - private final List sortOrders; - private final Optional temporalColumnHandle; - private final OptionalLong distributionId; - private final OptionalInt bucketCount; - private final List bucketColumnHandles; - private final boolean organized; - - @JsonCreator - public RaptorOutputTableHandle( - @JsonProperty("transactionId") long transactionId, - @JsonProperty("schemaName") String schemaName, - @JsonProperty("tableName") String tableName, - @JsonProperty("columnHandles") List columnHandles, - @JsonProperty("columnTypes") List columnTypes, - @JsonProperty("sortColumnHandles") List sortColumnHandles, - @JsonProperty("sortOrders") List sortOrders, - @JsonProperty("temporalColumnHandle") Optional temporalColumnHandle, - @JsonProperty("distributionId") OptionalLong distributionId, - @JsonProperty("bucketCount") OptionalInt bucketCount, - @JsonProperty("organized") boolean organized, - @JsonProperty("bucketColumnHandles") List bucketColumnHandles) - { - this.transactionId = transactionId; - this.schemaName = checkSchemaName(schemaName); - this.tableName = checkTableName(tableName); - this.columnHandles = ImmutableList.copyOf(requireNonNull(columnHandles, "columnHandles is null")); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.sortOrders = requireNonNull(sortOrders, "sortOrders is null"); - this.sortColumnHandles = requireNonNull(sortColumnHandles, "sortColumnHandles is null"); - this.temporalColumnHandle = requireNonNull(temporalColumnHandle, "temporalColumnHandle is null"); - this.distributionId = requireNonNull(distributionId, "distributionId is null"); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.bucketColumnHandles = ImmutableList.copyOf(requireNonNull(bucketColumnHandles, "bucketColumnHandles is null")); - this.organized = organized; - } - - @JsonProperty - public long getTransactionId() - { - return transactionId; - } - - @JsonProperty - public String getSchemaName() - { - return schemaName; - } - - @JsonProperty - public String getTableName() - { - return tableName; - } - - @JsonProperty - public List getColumnHandles() - { - return columnHandles; - } - - @JsonProperty - public List getColumnTypes() - { - return columnTypes; - } - - @JsonProperty - public List getSortColumnHandles() - { - return sortColumnHandles; - } - - @JsonProperty - public List getSortOrders() - { - return sortOrders; - } - - @JsonProperty - public Optional getTemporalColumnHandle() - { - return temporalColumnHandle; - } - - @JsonProperty - public OptionalLong getDistributionId() - { - return distributionId; - } - - @JsonProperty - public OptionalInt getBucketCount() - { - return bucketCount; - } - - @JsonProperty - public List getBucketColumnHandles() - { - return bucketColumnHandles; - } - - @JsonProperty - public boolean isOrganized() - { - return organized; - } - - @Override - public String toString() - { - return "raptor:" + schemaName + "." + tableName; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSink.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSink.java deleted file mode 100644 index 067002f98536..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSink.java +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.organization.TemporalFunction; -import io.trino.plugin.raptor.legacy.util.PageBuffer; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.PageSorter; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.type.Type; -import it.unimi.dsi.fastutil.longs.Long2ObjectMap; -import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.concurrent.CompletableFuture; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class RaptorPageSink - implements ConnectorPageSink -{ - private static final JsonCodec SHARD_INFO_CODEC = jsonCodec(ShardInfo.class); - - private final long transactionId; - private final StorageManager storageManager; - private final PageSorter pageSorter; - private final List columnIds; - private final List columnTypes; - private final List sortFields; - private final List sortOrders; - private final OptionalInt bucketCount; - private final int[] bucketFields; - private final long maxBufferBytes; - private final OptionalInt temporalColumnIndex; - private final Optional temporalColumnType; - - private final PageWriter pageWriter; - - public RaptorPageSink( - PageSorter pageSorter, - StorageManager storageManager, - long transactionId, - List columnIds, - List columnTypes, - List sortColumnIds, - List sortOrders, - OptionalInt bucketCount, - List bucketColumnIds, - Optional temporalColumnHandle, - DataSize maxBufferSize) - { - this.transactionId = transactionId; - this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); - this.columnIds = ImmutableList.copyOf(requireNonNull(columnIds, "columnIds is null")); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - this.maxBufferBytes = maxBufferSize.toBytes(); - - this.sortFields = sortColumnIds.stream().map(columnIds::indexOf).collect(toImmutableList()); - this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); - - this.bucketCount = bucketCount; - this.bucketFields = bucketColumnIds.stream().mapToInt(columnIds::indexOf).toArray(); - - if (temporalColumnHandle.isPresent() && columnIds.contains(temporalColumnHandle.get().getColumnId())) { - temporalColumnIndex = OptionalInt.of(columnIds.indexOf(temporalColumnHandle.get().getColumnId())); - temporalColumnType = Optional.of(columnTypes.get(temporalColumnIndex.getAsInt())); - checkArgument(temporalColumnType.get() == DATE || temporalColumnType.get().equals(TIMESTAMP_MILLIS), - "temporalColumnType can only be DATE or TIMESTAMP"); - } - else { - temporalColumnIndex = OptionalInt.empty(); - temporalColumnType = Optional.empty(); - } - - this.pageWriter = (bucketCount.isPresent() || temporalColumnIndex.isPresent()) ? new PartitionedPageWriter() : new SimplePageWriter(); - } - - @Override - public CompletableFuture appendPage(Page page) - { - if (page.getPositionCount() == 0) { - return NOT_BLOCKED; - } - - pageWriter.appendPage(page); - return NOT_BLOCKED; - } - - @Override - public CompletableFuture> finish() - { - List>> futureSlices = pageWriter.getPageBuffers().stream().map(pageBuffer -> { - pageBuffer.flush(); - CompletableFuture> futureShards = pageBuffer.getStoragePageSink().commit(); - return futureShards.thenApply(shards -> shards.stream() - .map(shard -> Slices.wrappedBuffer(SHARD_INFO_CODEC.toJsonBytes(shard))) - .collect(toList())); - }).collect(toList()); - - return allAsList(futureSlices).thenApply(lists -> lists.stream() - .flatMap(Collection::stream) - .collect(toList())); - } - - @Override - public void abort() - { - RuntimeException error = new RuntimeException("Exception during rollback"); - for (PageBuffer pageBuffer : pageWriter.getPageBuffers()) { - try { - pageBuffer.getStoragePageSink().rollback(); - } - catch (Throwable t) { - // Self-suppression not permitted - if (error != t) { - error.addSuppressed(t); - } - } - } - if (error.getSuppressed().length > 0) { - throw error; - } - } - - private PageBuffer createPageBuffer(OptionalInt bucketNumber) - { - return new PageBuffer( - maxBufferBytes, - storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, true), - columnTypes, - sortFields, - sortOrders, - pageSorter); - } - - private interface PageWriter - { - void appendPage(Page page); - - List getPageBuffers(); - } - - private class SimplePageWriter - implements PageWriter - { - private final PageBuffer pageBuffer = createPageBuffer(OptionalInt.empty()); - - @Override - public void appendPage(Page page) - { - pageBuffer.add(page); - } - - @Override - public List getPageBuffers() - { - return ImmutableList.of(pageBuffer); - } - } - - private class PartitionedPageWriter - implements PageWriter - { - private final Optional bucketFunction; - private final Long2ObjectMap pageStores = new Long2ObjectOpenHashMap<>(); - - public PartitionedPageWriter() - { - checkArgument(temporalColumnIndex.isPresent() == temporalColumnType.isPresent(), - "temporalColumnIndex and temporalColumnType must be both present or absent"); - - List bucketTypes = Arrays.stream(bucketFields) - .mapToObj(columnTypes::get) - .collect(toList()); - - this.bucketFunction = bucketCount.isPresent() ? Optional.of(new RaptorBucketFunction(bucketCount.getAsInt(), bucketTypes)) : Optional.empty(); - } - - @Override - public void appendPage(Page page) - { - Block temporalBlock = temporalColumnIndex.isPresent() ? page.getBlock(temporalColumnIndex.getAsInt()) : null; - - Page bucketArgs = bucketFunction.isPresent() ? getBucketArgsPage(page) : null; - - for (int position = 0; position < page.getPositionCount(); position++) { - int bucket = bucketFunction.isPresent() ? bucketFunction.get().getBucket(bucketArgs, position) : 0; - int day = temporalColumnType.isPresent() ? TemporalFunction.getDay(temporalColumnType.get(), temporalBlock, position) : 0; - - long partition = (((long) bucket) << 32) | (day & 0xFFFF_FFFFL); - PageStore store = pageStores.get(partition); - if (store == null) { - OptionalInt bucketNumber = bucketFunction.isPresent() ? OptionalInt.of(bucket) : OptionalInt.empty(); - PageBuffer buffer = createPageBuffer(bucketNumber); - store = new PageStore(buffer, columnTypes); - pageStores.put(partition, store); - } - - store.appendPosition(page, position); - } - - flushIfNecessary(); - } - - private Page getBucketArgsPage(Page page) - { - Block[] blocks = new Block[bucketFields.length]; - for (int i = 0; i < bucketFields.length; i++) { - blocks[i] = page.getBlock(bucketFields[i]); - } - return new Page(page.getPositionCount(), blocks); - } - - @Override - public List getPageBuffers() - { - ImmutableList.Builder list = ImmutableList.builder(); - for (PageStore store : pageStores.values()) { - store.flushToPageBuffer(); - store.getPageBuffer().flush(); - list.add(store.getPageBuffer()); - } - return list.build(); - } - - private void flushIfNecessary() - { - long totalBytes = 0; - long maxBytes = 0; - PageBuffer maxBuffer = null; - - for (PageStore store : pageStores.values()) { - long bytes = store.getUsedMemoryBytes(); - totalBytes += bytes; - - if ((maxBuffer == null) || (bytes > maxBytes)) { - maxBuffer = store.getPageBuffer(); - maxBytes = bytes; - } - } - - if ((totalBytes > maxBufferBytes) && (maxBuffer != null)) { - maxBuffer.flush(); - } - } - } - - private static class PageStore - { - private final PageBuffer pageBuffer; - private final PageBuilder pageBuilder; - - public PageStore(PageBuffer pageBuffer, List columnTypes) - { - this.pageBuffer = requireNonNull(pageBuffer, "pageBuffer is null"); - this.pageBuilder = new PageBuilder(columnTypes); - } - - public long getUsedMemoryBytes() - { - return pageBuilder.getSizeInBytes() + pageBuffer.getUsedMemoryBytes(); - } - - public PageBuffer getPageBuffer() - { - return pageBuffer; - } - - public void appendPosition(Page page, int position) - { - pageBuilder.declarePosition(); - for (int channel = 0; channel < page.getChannelCount(); channel++) { - Block block = page.getBlock(channel); - BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(channel); - pageBuilder.getType(channel).appendTo(block, position, blockBuilder); - } - - if (pageBuilder.isFull()) { - flushToPageBuffer(); - } - } - - public void flushToPageBuffer() - { - if (!pageBuilder.isEmpty()) { - pageBuffer.add(pageBuilder.build()); - pageBuilder.reset(); - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSinkProvider.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSinkProvider.java deleted file mode 100644 index 7a32548b4fd9..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSinkProvider.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.inject.Inject; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.PageSorter; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorMergeSink; -import io.trino.spi.connector.ConnectorMergeTableHandle; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.ConnectorPageSinkId; -import io.trino.spi.connector.ConnectorPageSinkProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTransactionHandle; - -import java.util.List; - -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class RaptorPageSinkProvider - implements ConnectorPageSinkProvider -{ - private final StorageManager storageManager; - private final PageSorter pageSorter; - private final DataSize maxBufferSize; - - @Inject - public RaptorPageSinkProvider(StorageManager storageManager, PageSorter pageSorter, StorageManagerConfig config) - { - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); - this.maxBufferSize = config.getMaxBufferSize(); - } - - @Override - public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle tableHandle, ConnectorPageSinkId pageSinkId) - { - RaptorOutputTableHandle handle = (RaptorOutputTableHandle) tableHandle; - return new RaptorPageSink( - pageSorter, - storageManager, - handle.getTransactionId(), - toColumnIds(handle.getColumnHandles()), - handle.getColumnTypes(), - toColumnIds(handle.getSortColumnHandles()), - handle.getSortOrders(), - handle.getBucketCount(), - toColumnIds(handle.getBucketColumnHandles()), - handle.getTemporalColumnHandle(), - maxBufferSize); - } - - @Override - public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle tableHandle, ConnectorPageSinkId pageSinkId) - { - RaptorInsertTableHandle handle = (RaptorInsertTableHandle) tableHandle; - return new RaptorPageSink( - pageSorter, - storageManager, - handle.getTransactionId(), - toColumnIds(handle.getColumnHandles()), - handle.getColumnTypes(), - toColumnIds(handle.getSortColumnHandles()), - handle.getSortOrders(), - handle.getBucketCount(), - toColumnIds(handle.getBucketColumnHandles()), - handle.getTemporalColumnHandle(), - maxBufferSize); - } - - @Override - public ConnectorMergeSink createMergeSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorMergeTableHandle mergeHandle, ConnectorPageSinkId pageSinkId) - { - RaptorMergeTableHandle merge = (RaptorMergeTableHandle) mergeHandle; - ConnectorPageSink pageSink = createPageSink(transactionHandle, session, merge.getInsertTableHandle(), pageSinkId); - long transactionId = merge.getInsertTableHandle().getTransactionId(); - int columnCount = merge.getInsertTableHandle().getColumnHandles().size(); - return new RaptorMergeSink(pageSink, storageManager, transactionId, columnCount); - } - - private static List toColumnIds(List columnHandles) - { - return columnHandles.stream().map(RaptorColumnHandle::getColumnId).collect(toList()); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSourceProvider.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSourceProvider.java deleted file mode 100644 index e266d8562e73..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPageSourceProvider.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.inject.Inject; -import io.trino.orc.OrcReaderOptions; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.util.ConcatPageSource; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.ConnectorPageSourceProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; - -import java.util.Iterator; -import java.util.List; -import java.util.OptionalInt; -import java.util.UUID; - -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderMaxMergeDistance; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderMaxReadSize; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderStreamBufferSize; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderTinyStripeThreshold; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.isReaderLazyReadSmallRanges; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class RaptorPageSourceProvider - implements ConnectorPageSourceProvider -{ - private final StorageManager storageManager; - - @Inject - public RaptorPageSourceProvider(StorageManager storageManager) - { - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - } - - @Override - public ConnectorPageSource createPageSource( - ConnectorTransactionHandle transaction, - ConnectorSession session, - ConnectorSplit split, - ConnectorTableHandle table, - List columns, - DynamicFilter dynamicFilter) - { - RaptorSplit raptorSplit = (RaptorSplit) split; - RaptorTableHandle raptorTable = (RaptorTableHandle) table; - - OptionalInt bucketNumber = raptorSplit.getBucketNumber(); - TupleDomain predicate = raptorTable.getConstraint(); - OrcReaderOptions options = new OrcReaderOptions() - .withMaxMergeDistance(getReaderMaxMergeDistance(session)) - .withMaxBufferSize(getReaderMaxReadSize(session)) - .withStreamBufferSize(getReaderStreamBufferSize(session)) - .withTinyStripeThreshold(getReaderTinyStripeThreshold(session)) - .withLazyReadSmallRanges(isReaderLazyReadSmallRanges(session)); - - if (raptorSplit.getShardUuids().size() == 1) { - UUID shardUuid = raptorSplit.getShardUuids().iterator().next(); - return createPageSource(shardUuid, bucketNumber, columns, predicate, options); - } - - Iterator iterator = raptorSplit.getShardUuids().stream() - .map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, options)) - .iterator(); - - return new ConcatPageSource(iterator); - } - - private ConnectorPageSource createPageSource( - UUID shardUuid, - OptionalInt bucketNumber, - List columns, - TupleDomain predicate, - OrcReaderOptions orcReaderOptions) - { - List columnHandles = columns.stream().map(RaptorColumnHandle.class::cast).collect(toList()); - List columnIds = columnHandles.stream().map(RaptorColumnHandle::getColumnId).collect(toList()); - List columnTypes = columnHandles.stream().map(RaptorColumnHandle::getColumnType).collect(toList()); - - return storageManager.getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, predicate, orcReaderOptions); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPartitioningHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPartitioningHandle.java deleted file mode 100644 index ca6055e1bcd2..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPartitioningHandle.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ConnectorPartitioningHandle; - -import java.util.List; -import java.util.Objects; - -import static java.util.Objects.requireNonNull; - -public class RaptorPartitioningHandle - implements ConnectorPartitioningHandle -{ - private final long distributionId; - private final List bucketToNode; - - @JsonCreator - public RaptorPartitioningHandle( - @JsonProperty("distributionId") long distributionId, - @JsonProperty("bucketToNode") List bucketToNode) - { - this.distributionId = distributionId; - this.bucketToNode = ImmutableList.copyOf(requireNonNull(bucketToNode, "bucketToNode is null")); - } - - @JsonProperty - public long getDistributionId() - { - return distributionId; - } - - @JsonProperty - public List getBucketToNode() - { - return bucketToNode; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if ((o == null) || (getClass() != o.getClass())) { - return false; - } - RaptorPartitioningHandle that = (RaptorPartitioningHandle) o; - return (distributionId == that.distributionId) && - Objects.equals(bucketToNode, that.bucketToNode); - } - - @Override - public int hashCode() - { - return Objects.hash(distributionId, bucketToNode); - } - - @Override - public String toString() - { - return String.valueOf(distributionId); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPlugin.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPlugin.java deleted file mode 100644 index d2159a946dcc..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorPlugin.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.inject.Module; -import io.trino.plugin.raptor.legacy.metadata.DatabaseMetadataModule; -import io.trino.spi.Plugin; -import io.trino.spi.connector.ConnectorFactory; - -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.util.Objects.requireNonNull; - -public class RaptorPlugin - implements Plugin -{ - private final String name; - private final Module metadataModule; - private final Map backupProviders; - - public RaptorPlugin() - { - this("raptor_legacy", new DatabaseMetadataModule(), ImmutableMap.of()); - } - - public RaptorPlugin(String name, Module metadataModule, Map backupProviders) - { - checkArgument(!isNullOrEmpty(name), "name is null or empty"); - this.name = name; - this.metadataModule = requireNonNull(metadataModule, "metadataModule is null"); - this.backupProviders = ImmutableMap.copyOf(requireNonNull(backupProviders, "backupProviders is null")); - } - - @Override - public Iterable getConnectorFactories() - { - return ImmutableList.of(new RaptorConnectorFactory(name, metadataModule, backupProviders)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSessionProperties.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSessionProperties.java deleted file mode 100644 index f9f47e1edf25..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSessionProperties.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.session.PropertyMetadata; - -import java.util.List; -import java.util.Optional; - -import static io.trino.plugin.base.session.PropertyMetadataUtil.dataSizeProperty; -import static io.trino.spi.session.PropertyMetadata.booleanProperty; -import static io.trino.spi.session.PropertyMetadata.integerProperty; -import static io.trino.spi.session.PropertyMetadata.stringProperty; - -public class RaptorSessionProperties -{ - private static final String EXTERNAL_BATCH_ID = "external_batch_id"; - - private static final String READER_MAX_MERGE_DISTANCE = "reader_max_merge_distance"; - private static final String READER_MAX_READ_SIZE = "reader_max_read_size"; - private static final String READER_STREAM_BUFFER_SIZE = "reader_stream_buffer_size"; - private static final String READER_TINY_STRIPE_THRESHOLD = "reader_tiny_stripe_threshold"; - private static final String READER_LAZY_READ_SMALL_RANGES = "reader_lazy_read_small_ranges"; - private static final String ONE_SPLIT_PER_BUCKET_THRESHOLD = "one_split_per_bucket_threshold"; - - private final List> sessionProperties; - - @Inject - public RaptorSessionProperties(StorageManagerConfig config) - { - sessionProperties = ImmutableList.of( - stringProperty( - EXTERNAL_BATCH_ID, - "Two-phase commit batch ID", - null, - true), - dataSizeProperty( - READER_MAX_MERGE_DISTANCE, - "Reader: Maximum size of gap between two reads to merge into a single read", - config.getOrcMaxMergeDistance(), - false), - dataSizeProperty( - READER_MAX_READ_SIZE, - "Reader: Maximum size of a single read", - config.getOrcMaxReadSize(), - false), - dataSizeProperty( - READER_STREAM_BUFFER_SIZE, - "Reader: Size of buffer for streaming reads", - config.getOrcStreamBufferSize(), - false), - dataSizeProperty( - READER_TINY_STRIPE_THRESHOLD, - "Reader: Threshold below which an ORC stripe or file will read in its entirety", - config.getOrcTinyStripeThreshold(), - false), - booleanProperty( - READER_LAZY_READ_SMALL_RANGES, - "Experimental: Reader: Read small file segments lazily", - config.isOrcLazyReadSmallRanges(), - false), - integerProperty( - ONE_SPLIT_PER_BUCKET_THRESHOLD, - "Experimental: Maximum bucket count at which to produce multiple splits per bucket", - config.getOneSplitPerBucketThreshold(), - false)); - } - - public List> getSessionProperties() - { - return sessionProperties; - } - - public static Optional getExternalBatchId(ConnectorSession session) - { - return Optional.ofNullable(session.getProperty(EXTERNAL_BATCH_ID, String.class)); - } - - public static DataSize getReaderMaxMergeDistance(ConnectorSession session) - { - return session.getProperty(READER_MAX_MERGE_DISTANCE, DataSize.class); - } - - public static DataSize getReaderMaxReadSize(ConnectorSession session) - { - return session.getProperty(READER_MAX_READ_SIZE, DataSize.class); - } - - public static DataSize getReaderStreamBufferSize(ConnectorSession session) - { - return session.getProperty(READER_STREAM_BUFFER_SIZE, DataSize.class); - } - - public static DataSize getReaderTinyStripeThreshold(ConnectorSession session) - { - return session.getProperty(READER_TINY_STRIPE_THRESHOLD, DataSize.class); - } - - public static boolean isReaderLazyReadSmallRanges(ConnectorSession session) - { - return session.getProperty(READER_LAZY_READ_SMALL_RANGES, Boolean.class); - } - - public static int getOneSplitPerBucketThreshold(ConnectorSession session) - { - return session.getProperty(ONE_SPLIT_PER_BUCKET_THRESHOLD, Integer.class); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplit.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplit.java deleted file mode 100644 index f2198b73e973..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplit.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.trino.spi.HostAddress; -import io.trino.spi.connector.ConnectorSplit; - -import java.util.List; -import java.util.Map; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.airlift.slice.SizeOf.estimatedSizeOf; -import static io.airlift.slice.SizeOf.instanceSize; -import static io.airlift.slice.SizeOf.sizeOf; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.joining; - -public class RaptorSplit - implements ConnectorSplit -{ - private static final int INSTANCE_SIZE = instanceSize(RaptorSplit.class); - private static final int UUID_INSTANCE_SIZE = instanceSize(UUID.class); - - private final Set shardUuids; - private final OptionalInt bucketNumber; - private final List addresses; - - @JsonCreator - public RaptorSplit( - @JsonProperty("shardUuids") Set shardUuids, - @JsonProperty("bucketNumber") OptionalInt bucketNumber) - { - this(shardUuids, bucketNumber, ImmutableList.of()); - } - - public RaptorSplit(UUID shardUuid, List addresses) - { - this(ImmutableSet.of(shardUuid), OptionalInt.empty(), addresses); - } - - public RaptorSplit(Set shardUuids, int bucketNumber, HostAddress address) - { - this(shardUuids, OptionalInt.of(bucketNumber), ImmutableList.of(address)); - } - - private RaptorSplit(Set shardUuids, OptionalInt bucketNumber, List addresses) - { - this.shardUuids = ImmutableSet.copyOf(requireNonNull(shardUuids, "shardUuids is null")); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.addresses = ImmutableList.copyOf(requireNonNull(addresses, "addresses is null")); - } - - @Override - public boolean isRemotelyAccessible() - { - return false; - } - - @Override - public List getAddresses() - { - return addresses; - } - - @JsonProperty - public Set getShardUuids() - { - return shardUuids; - } - - @JsonProperty - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - @Override - public Map getSplitInfo() - { - return ImmutableMap.of( - "addresses", addresses.stream().map(HostAddress::toString).collect(joining(",")), - "bucketNumber", bucketNumber.isPresent() ? String.valueOf(bucketNumber.getAsInt()) : "", - "shardUuids", shardUuids.stream().map(UUID::toString).collect(joining(","))); - } - - @Override - public long getRetainedSizeInBytes() - { - return INSTANCE_SIZE - + estimatedSizeOf(shardUuids, value -> UUID_INSTANCE_SIZE) - + sizeOf(bucketNumber) - + estimatedSizeOf(addresses, HostAddress::getRetainedSizeInBytes); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuids", shardUuids) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .add("hosts", addresses) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplitManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplitManager.java deleted file mode 100644 index e480bdd2b67d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorSplitManager.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.backup.BackupService; -import io.trino.plugin.raptor.legacy.metadata.BucketShards; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardNodes; -import io.trino.plugin.raptor.legacy.util.SynchronizedResultIterator; -import io.trino.spi.HostAddress; -import io.trino.spi.Node; -import io.trino.spi.TrinoException; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorSplitSource; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.predicate.TupleDomain; -import jakarta.annotation.PreDestroy; -import org.jdbi.v3.core.result.ResultIterator; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.function.Supplier; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.collect.Maps.uniqueIndex; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_NO_HOST_FOR_SHARD; -import static io.trino.plugin.raptor.legacy.RaptorSessionProperties.getOneSplitPerBucketThreshold; -import static io.trino.spi.StandardErrorCode.NO_NODES_AVAILABLE; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.supplyAsync; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.stream.Collectors.toSet; - -public class RaptorSplitManager - implements ConnectorSplitManager -{ - private final NodeSupplier nodeSupplier; - private final ShardManager shardManager; - private final boolean backupAvailable; - private final ExecutorService executor; - - @Inject - public RaptorSplitManager(CatalogName catalogName, NodeSupplier nodeSupplier, ShardManager shardManager, BackupService backupService) - { - this(catalogName, nodeSupplier, shardManager, backupService.isBackupAvailable()); - } - - public RaptorSplitManager(CatalogName catalogName, NodeSupplier nodeSupplier, ShardManager shardManager, boolean backupAvailable) - { - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.backupAvailable = backupAvailable; - this.executor = newCachedThreadPool(daemonThreadsNamed("raptor-split-" + catalogName + "-%s")); - } - - @PreDestroy - public void destroy() - { - executor.shutdownNow(); - } - - @Override - public ConnectorSplitSource getSplits( - ConnectorTransactionHandle transaction, - ConnectorSession session, - ConnectorTableHandle handle, - DynamicFilter dynamicFilter, - Constraint constraint) - { - RaptorTableHandle table = (RaptorTableHandle) handle; - long tableId = table.getTableId(); - boolean bucketed = table.getBucketCount().isPresent(); - boolean merged = bucketed && (table.getBucketCount().getAsInt() >= getOneSplitPerBucketThreshold(session)); - Optional> bucketToNode = table.getBucketAssignments(); - verify(bucketed == bucketToNode.isPresent(), "mismatched bucketCount and bucketToNode presence"); - return new RaptorSplitSource(tableId, merged, table.getConstraint(), bucketToNode); - } - - private static List getAddressesForNodes(Map nodeMap, Iterable nodeIdentifiers) - { - ImmutableList.Builder nodes = ImmutableList.builder(); - for (String id : nodeIdentifiers) { - Node node = nodeMap.get(id); - if (node != null) { - nodes.add(node.getHostAndPort()); - } - } - return nodes.build(); - } - - private static T selectRandom(Iterable elements) - { - List list = ImmutableList.copyOf(elements); - return list.get(ThreadLocalRandom.current().nextInt(list.size())); - } - - private class RaptorSplitSource - implements ConnectorSplitSource - { - private final Map nodesById = uniqueIndex(nodeSupplier.getWorkerNodes(), Node::getNodeIdentifier); - private final long tableId; - private final Optional> bucketToNode; - private final ResultIterator iterator; - - @GuardedBy("this") - private CompletableFuture future; - - public RaptorSplitSource( - long tableId, - boolean merged, - TupleDomain effectivePredicate, - Optional> bucketToNode) - { - this.tableId = tableId; - this.bucketToNode = requireNonNull(bucketToNode, "bucketToNode is null"); - - ResultIterator iterator; - if (bucketToNode.isPresent()) { - iterator = shardManager.getShardNodesBucketed(tableId, merged, bucketToNode.get(), effectivePredicate); - } - else { - iterator = shardManager.getShardNodes(tableId, effectivePredicate); - } - this.iterator = new SynchronizedResultIterator<>(iterator); - } - - @Override - public synchronized CompletableFuture getNextBatch(int maxSize) - { - checkState((future == null) || future.isDone(), "previous batch not completed"); - future = supplyAsync(batchSupplier(maxSize), executor); - return future; - } - - @Override - public synchronized void close() - { - if (future != null) { - future.cancel(true); - future = null; - } - executor.execute(iterator::close); - } - - @Override - public boolean isFinished() - { - return !iterator.hasNext(); - } - - private Supplier batchSupplier(int maxSize) - { - return () -> { - ImmutableList.Builder list = ImmutableList.builder(); - for (int i = 0; i < maxSize; i++) { - if (Thread.currentThread().isInterrupted()) { - throw new RuntimeException("Split batch fetch was interrupted"); - } - if (!iterator.hasNext()) { - break; - } - list.add(createSplit(iterator.next())); - } - return new ConnectorSplitBatch(list.build(), isFinished()); - }; - } - - private ConnectorSplit createSplit(BucketShards bucketShards) - { - if (bucketShards.getBucketNumber().isPresent()) { - return createBucketSplit(bucketShards.getBucketNumber().getAsInt(), bucketShards.getShards()); - } - - verify(bucketShards.getShards().size() == 1, "wrong shard count for non-bucketed table"); - ShardNodes shard = getOnlyElement(bucketShards.getShards()); - UUID shardId = shard.getShardUuid(); - Set nodeIds = shard.getNodeIdentifiers(); - - List addresses = getAddressesForNodes(nodesById, nodeIds); - if (addresses.isEmpty()) { - if (!backupAvailable) { - throw new TrinoException(RAPTOR_NO_HOST_FOR_SHARD, format("No host for shard %s found: %s", shardId, nodeIds)); - } - - // Pick a random node and optimistically assign the shard to it. - // That node will restore the shard from the backup location. - Set availableNodes = nodeSupplier.getWorkerNodes(); - if (availableNodes.isEmpty()) { - throw new TrinoException(NO_NODES_AVAILABLE, "No nodes available to run query"); - } - Node node = selectRandom(availableNodes); - shardManager.replaceShardAssignment(tableId, shardId, node.getNodeIdentifier(), true); - addresses = ImmutableList.of(node.getHostAndPort()); - } - - return new RaptorSplit(shardId, addresses); - } - - private ConnectorSplit createBucketSplit(int bucketNumber, Set shards) - { - // Bucket splits contain all the shards for the bucket - // and run on the node assigned to the bucket. - - String nodeId = bucketToNode.get().get(bucketNumber); - Node node = nodesById.get(nodeId); - if (node == null) { - throw new TrinoException(NO_NODES_AVAILABLE, "Node for bucket is offline: " + nodeId); - } - - Set shardUuids = shards.stream() - .map(ShardNodes::getShardUuid) - .collect(toSet()); - HostAddress address = node.getHostAndPort(); - - return new RaptorSplit(shardUuids, bucketNumber, address); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableHandle.java deleted file mode 100644 index 4bb583bc3771..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableHandle.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.predicate.TupleDomain; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.raptor.legacy.util.MetadataUtil.checkSchemaName; -import static io.trino.plugin.raptor.legacy.util.MetadataUtil.checkTableName; -import static java.util.Objects.requireNonNull; - -public final class RaptorTableHandle - implements ConnectorTableHandle -{ - private final String schemaName; - private final String tableName; - private final long tableId; - private final Optional distributionId; - private final Optional distributionName; - private final OptionalInt bucketCount; - private final boolean organized; - private final TupleDomain constraint; - private final Optional> bucketAssignments; - - @JsonCreator - public RaptorTableHandle( - @JsonProperty("schemaName") String schemaName, - @JsonProperty("tableName") String tableName, - @JsonProperty("tableId") long tableId, - @JsonProperty("distributionId") Optional distributionId, - @JsonProperty("distributionName") Optional distributionName, - @JsonProperty("bucketCount") OptionalInt bucketCount, - @JsonProperty("organized") boolean organized, - @JsonProperty("constraint") TupleDomain constraint, - // this field will not be in the JSON, but keep it here to avoid duplicating the constructor - @JsonProperty("bucketAssignments") Optional> bucketAssignments) - { - this.schemaName = checkSchemaName(schemaName); - this.tableName = checkTableName(tableName); - - checkArgument(tableId > 0, "tableId must be greater than zero"); - this.tableId = tableId; - - this.distributionName = requireNonNull(distributionName, "distributionName is null"); - this.distributionId = requireNonNull(distributionId, "distributionId is null"); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.organized = organized; - - this.constraint = requireNonNull(constraint, "constraint is null"); - this.bucketAssignments = bucketAssignments.map(ImmutableList::copyOf); - } - - public boolean isBucketed() - { - return this.distributionId.isPresent(); - } - - @JsonProperty - public String getSchemaName() - { - return schemaName; - } - - @JsonProperty - public String getTableName() - { - return tableName; - } - - @JsonProperty - public long getTableId() - { - return tableId; - } - - @JsonProperty - public Optional getDistributionId() - { - return distributionId; - } - - @JsonProperty - public Optional getDistributionName() - { - return distributionName; - } - - @JsonProperty - public OptionalInt getBucketCount() - { - return bucketCount; - } - - @JsonProperty - public boolean isOrganized() - { - return organized; - } - - @JsonProperty - public TupleDomain getConstraint() - { - return constraint; - } - - @JsonIgnore - public Optional> getBucketAssignments() - { - return bucketAssignments; - } - - @Override - public String toString() - { - return schemaName + ":" + tableName + ":" + tableId; - } - - @Override - public int hashCode() - { - return Objects.hash(schemaName, tableName, tableId, distributionId, distributionName, bucketCount, organized, constraint, bucketAssignments); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - RaptorTableHandle other = (RaptorTableHandle) obj; - return Objects.equals(this.schemaName, other.schemaName) && - Objects.equals(this.tableName, other.tableName) && - this.tableId == other.tableId && - Objects.equals(this.distributionId, other.distributionId) && - Objects.equals(this.distributionName, other.distributionName) && - Objects.equals(this.bucketCount, other.bucketCount) && - this.organized == other.organized && - Objects.equals(this.constraint, other.constraint) && - Objects.equals(this.bucketAssignments, other.bucketAssignments); - } - - @JsonIgnore - public Optional getPartitioningHandle() - { - return distributionId.map(id -> new RaptorPartitioningHandle(id, bucketAssignments.get())); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableProperties.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableProperties.java deleted file mode 100644 index 293f89a602f2..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTableProperties.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.trino.spi.session.PropertyMetadata; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeSignatureParameter; - -import java.util.List; -import java.util.Map; -import java.util.OptionalInt; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.spi.session.PropertyMetadata.booleanProperty; -import static io.trino.spi.session.PropertyMetadata.integerProperty; -import static io.trino.spi.type.StandardTypes.ARRAY; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static java.util.Locale.ENGLISH; - -public class RaptorTableProperties -{ - public static final String ORDERING_PROPERTY = "ordering"; - public static final String TEMPORAL_COLUMN_PROPERTY = "temporal_column"; - public static final String BUCKET_COUNT_PROPERTY = "bucket_count"; - public static final String BUCKETED_ON_PROPERTY = "bucketed_on"; - public static final String DISTRIBUTION_NAME_PROPERTY = "distribution_name"; - public static final String ORGANIZED_PROPERTY = "organized"; - - private final List> tableProperties; - - @Inject - public RaptorTableProperties(TypeManager typeManager) - { - tableProperties = ImmutableList.>builder() - .add(stringListSessionProperty( - typeManager, - ORDERING_PROPERTY, - "Sort order for each shard of the table")) - .add(lowerCaseStringSessionProperty( - TEMPORAL_COLUMN_PROPERTY, - "Temporal column of the table")) - .add(integerProperty( - BUCKET_COUNT_PROPERTY, - "Number of buckets into which to divide the table", - null, - false)) - .add(stringListSessionProperty( - typeManager, - BUCKETED_ON_PROPERTY, - "Table columns on which to bucket the table")) - .add(lowerCaseStringSessionProperty( - DISTRIBUTION_NAME_PROPERTY, - "Shared distribution name for colocated tables")) - .add(booleanProperty( - ORGANIZED_PROPERTY, - "Keep the table organized using the sort order", - null, - false)) - .build(); - } - - public List> getTableProperties() - { - return tableProperties; - } - - public static List getSortColumns(Map tableProperties) - { - return stringList(tableProperties.get(ORDERING_PROPERTY)); - } - - public static String getTemporalColumn(Map tableProperties) - { - return (String) tableProperties.get(TEMPORAL_COLUMN_PROPERTY); - } - - public static OptionalInt getBucketCount(Map tableProperties) - { - Integer value = (Integer) tableProperties.get(BUCKET_COUNT_PROPERTY); - return (value != null) ? OptionalInt.of(value) : OptionalInt.empty(); - } - - public static List getBucketColumns(Map tableProperties) - { - return stringList(tableProperties.get(BUCKETED_ON_PROPERTY)); - } - - public static String getDistributionName(Map tableProperties) - { - return (String) tableProperties.get(DISTRIBUTION_NAME_PROPERTY); - } - - public static boolean isOrganized(Map tableProperties) - { - Boolean value = (Boolean) tableProperties.get(ORGANIZED_PROPERTY); - return (value == null) ? false : value; - } - - public static PropertyMetadata lowerCaseStringSessionProperty(String name, String description) - { - return new PropertyMetadata<>( - name, - description, - createUnboundedVarcharType(), - String.class, - null, - false, - value -> ((String) value).toLowerCase(ENGLISH), - value -> value); - } - - private static PropertyMetadata stringListSessionProperty(TypeManager typeManager, String name, String description) - { - return new PropertyMetadata<>( - name, - description, - typeManager.getParameterizedType(ARRAY, ImmutableList.of(TypeSignatureParameter.typeParameter(createUnboundedVarcharType().getTypeSignature()))), - List.class, - ImmutableList.of(), - false, - value -> stringList(value).stream() - .map(s -> s.toLowerCase(ENGLISH)) - .collect(toImmutableList()), - value -> value); - } - - @SuppressWarnings("unchecked") - private static List stringList(Object value) - { - return (value == null) ? ImmutableList.of() : ((List) value); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTransactionHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTransactionHandle.java deleted file mode 100644 index 66f058dc30cc..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorTransactionHandle.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.connector.ConnectorTransactionHandle; - -import java.util.Objects; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class RaptorTransactionHandle - implements ConnectorTransactionHandle -{ - private final UUID uuid; - - public RaptorTransactionHandle() - { - this(UUID.randomUUID()); - } - - @JsonCreator - public RaptorTransactionHandle(@JsonProperty("uuid") UUID uuid) - { - this.uuid = requireNonNull(uuid, "uuid is null"); - } - - @JsonProperty - public UUID getUuid() - { - return uuid; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - RaptorTransactionHandle other = (RaptorTransactionHandle) obj; - return Objects.equals(uuid, other.uuid); - } - - @Override - public int hashCode() - { - return Objects.hash(uuid); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("uuid", uuid) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateFunction.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateFunction.java deleted file mode 100644 index e584d991ed65..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateFunction.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.airlift.slice.Slice; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.RowBlock; -import io.trino.spi.block.SqlRow; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.type.UuidType; - -public class RaptorUnbucketedUpdateFunction - implements BucketFunction -{ - private final int bucketCount; - - public RaptorUnbucketedUpdateFunction(int bucketCount) - { - this.bucketCount = bucketCount; - } - - @Override - public int getBucket(Page page, int position) - { - Block block = page.getBlock(0); - SqlRow row = ((RowBlock) block.getUnderlyingValueBlock()).getRow(block.getUnderlyingValuePosition(position)); - Slice uuid = UuidType.UUID.getSlice(row.getRawFieldBlock(1), row.getRawIndex()); // uuid field of row ID - return (uuid.hashCode() & Integer.MAX_VALUE) % bucketCount; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateHandle.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateHandle.java deleted file mode 100644 index 83e361b67fc1..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/RaptorUnbucketedUpdateHandle.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.spi.connector.ConnectorPartitioningHandle; - -public enum RaptorUnbucketedUpdateHandle - implements ConnectorPartitioningHandle -{ - INSTANCE -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupConfig.java deleted file mode 100644 index b081961f8f4e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupConfig.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; -import io.airlift.units.MaxDuration; -import io.airlift.units.MinDuration; -import jakarta.annotation.Nullable; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; - -import static java.util.concurrent.TimeUnit.MINUTES; - -public class BackupConfig -{ - private Duration timeout = new Duration(1, MINUTES); - private int timeoutThreads = 1000; - private String provider; - private int backupThreads = 5; - - @NotNull - @MinDuration("1s") - @MaxDuration("1h") - public Duration getTimeout() - { - return timeout; - } - - @Config("backup.timeout") - @ConfigDescription("Timeout for per-shard backup operations") - public BackupConfig setTimeout(Duration timeout) - { - this.timeout = timeout; - return this; - } - - @Min(1) - public int getTimeoutThreads() - { - return timeoutThreads; - } - - @Config("backup.timeout-threads") - @ConfigDescription("Maximum number of timeout threads for backup operations") - public BackupConfig setTimeoutThreads(int timeoutThreads) - { - this.timeoutThreads = timeoutThreads; - return this; - } - - @Nullable - public String getProvider() - { - return provider; - } - - @Config("backup.provider") - @ConfigDescription("Backup provider to use (supported types: file)") - public BackupConfig setProvider(String provider) - { - this.provider = provider; - return this; - } - - @Min(1) - public int getBackupThreads() - { - return backupThreads; - } - - @Config("backup.threads") - @ConfigDescription("Maximum number of shards to backup at once") - public BackupConfig setBackupThreads(int backupThreads) - { - this.backupThreads = backupThreads; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupManager.java deleted file mode 100644 index 258a78496f16..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupManager.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.io.Files; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.storage.BackupStats; -import io.trino.plugin.raptor.legacy.storage.StorageService; -import io.trino.spi.TrinoException; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Flatten; -import org.weakref.jmx.Managed; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.completedFuture; -import static java.util.concurrent.CompletableFuture.runAsync; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class BackupManager -{ - private static final Logger log = Logger.get(BackupManager.class); - - private final Optional backupStore; - private final StorageService storageService; - private final ExecutorService executorService; - - private final AtomicInteger pendingBackups = new AtomicInteger(); - private final BackupStats stats = new BackupStats(); - - @Inject - public BackupManager(Optional backupStore, StorageService storageService, BackupConfig config) - { - this(backupStore, storageService, config.getBackupThreads()); - } - - public BackupManager(Optional backupStore, StorageService storageService, int backupThreads) - { - checkArgument(backupThreads > 0, "backupThreads must be > 0"); - - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.storageService = requireNonNull(storageService, "storageService is null"); - this.executorService = newFixedThreadPool(backupThreads, daemonThreadsNamed("background-shard-backup-%s")); - } - - @PreDestroy - public void shutdown() - { - shutdownAndAwaitTermination(executorService, 10, SECONDS); - } - - public CompletableFuture submit(UUID uuid, File source) - { - requireNonNull(uuid, "uuid is null"); - requireNonNull(source, "source is null"); - - if (backupStore.isEmpty()) { - return completedFuture(null); - } - - // TODO: decrement when the running task is finished (not immediately on cancel) - pendingBackups.incrementAndGet(); - CompletableFuture future = runAsync(new BackgroundBackup(uuid, source), executorService); - future.whenComplete((none, throwable) -> pendingBackups.decrementAndGet()); - return future; - } - - private class BackgroundBackup - implements Runnable - { - private final UUID uuid; - private final File source; - private final long queuedTime = System.nanoTime(); - - public BackgroundBackup(UUID uuid, File source) - { - this.uuid = requireNonNull(uuid, "uuid is null"); - this.source = requireNonNull(source, "source is null"); - } - - @Override - public void run() - { - try { - stats.addQueuedTime(Duration.nanosSince(queuedTime)); - long start = System.nanoTime(); - - backupStore.get().backupShard(uuid, source); - stats.addCopyShardDataRate(DataSize.ofBytes(source.length()), Duration.nanosSince(start)); - - File restored = new File(storageService.getStagingFile(uuid) + ".validate"); - backupStore.get().restoreShard(uuid, restored); - - if (!filesEqual(source, restored)) { - stats.incrementBackupCorruption(); - - File quarantineBase = storageService.getQuarantineFile(uuid); - File quarantineOriginal = new File(quarantineBase.getPath() + ".original"); - File quarantineRestored = new File(quarantineBase.getPath() + ".restored"); - - log.error("Backup is corrupt after write. Quarantining local file: %s", quarantineBase); - if (!this.source.renameTo(quarantineOriginal) || !restored.renameTo(quarantineRestored)) { - log.warn("Quarantine of corrupt backup shard failed: %s", uuid); - } - - throw new TrinoException(RAPTOR_BACKUP_CORRUPTION, "Backup is corrupt after write: " + uuid); - } - - if (!restored.delete()) { - log.warn("Failed to delete staging file: %s", restored); - } - - stats.incrementBackupSuccess(); - } - catch (Throwable t) { - stats.incrementBackupFailure(); - throw t; - } - } - } - - @Managed - public int getPendingBackupCount() - { - return pendingBackups.get(); - } - - @Managed - @Flatten - public BackupStats getStats() - { - return stats; - } - - private static boolean filesEqual(File file1, File file2) - { - try { - return Files.equal(file1, file2); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupModule.java deleted file mode 100644 index 40847952a62b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupModule.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; -import com.google.inject.util.Providers; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.configuration.ConfigurationAwareModule; -import io.trino.spi.catalog.CatalogName; -import jakarta.annotation.Nullable; -import org.weakref.jmx.MBeanExporter; - -import java.util.Map; -import java.util.Optional; - -import static io.airlift.configuration.ConfigBinder.configBinder; - -public class BackupModule - extends AbstractConfigurationAwareModule -{ - private final Map providers; - - public BackupModule(Map providers) - { - this.providers = ImmutableMap.builder() - .put("file", new FileBackupModule()) - .put("http", new HttpBackupModule()) - .putAll(providers) - .buildOrThrow(); - } - - @Override - protected void setup(Binder binder) - { - configBinder(binder).bindConfig(BackupConfig.class); - - String provider = buildConfigObject(BackupConfig.class).getProvider(); - if (provider == null) { - binder.bind(BackupStore.class).toProvider(Providers.of(null)); - } - else { - Module module = providers.get(provider); - if (module == null) { - binder.addError("Unknown backup provider: %s", provider); - } - else if (module instanceof ConfigurationAwareModule) { - install(module); - } - else { - binder.install(module); - } - } - binder.bind(BackupService.class).to(BackupServiceManager.class).in(Scopes.SINGLETON); - } - - @Provides - @Singleton - private static Optional createBackupStore( - @Nullable BackupStore store, - LifeCycleManager lifeCycleManager, - MBeanExporter exporter, - CatalogName catalogName, - BackupConfig config) - { - if (store == null) { - return Optional.empty(); - } - - BackupStore proxy = new TimeoutBackupStore( - store, - catalogName.toString(), - config.getTimeout(), - config.getTimeoutThreads()); - - lifeCycleManager.addInstance(proxy); - - BackupStore managed = new ManagedBackupStore(proxy); - exporter.exportWithGeneratedName(managed, BackupStore.class, catalogName.toString()); - - return Optional.of(managed); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupOperationStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupOperationStats.java deleted file mode 100644 index 32d21d0d1488..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupOperationStats.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeStat; -import io.trino.spi.TrinoException; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.util.function.Supplier; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_NOT_FOUND; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_TIMEOUT; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class BackupOperationStats -{ - private final TimeStat time = new TimeStat(MILLISECONDS); - private final CounterStat successes = new CounterStat(); - private final CounterStat failures = new CounterStat(); - private final CounterStat timeouts = new CounterStat(); - - @Managed - @Nested - public TimeStat getTime() - { - return time; - } - - @Managed - @Nested - public CounterStat getSuccesses() - { - return successes; - } - - @Managed - @Nested - public CounterStat getFailures() - { - return failures; - } - - @Managed - @Nested - public CounterStat getTimeouts() - { - return timeouts; - } - - public void run(Runnable runnable) - { - run(() -> { - runnable.run(); - return null; - }); - } - - public V run(Supplier supplier) - { - try (TimeStat.BlockTimer _ = time.time()) { - V value = supplier.get(); - successes.update(1); - return value; - } - catch (TrinoException e) { - if (e.getErrorCode().equals(RAPTOR_BACKUP_NOT_FOUND.toErrorCode())) { - successes.update(1); - } - else if (e.getErrorCode().equals(RAPTOR_BACKUP_TIMEOUT.toErrorCode())) { - timeouts.update(1); - } - else { - failures.update(1); - } - throw e; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupService.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupService.java deleted file mode 100644 index 5d8a66f3b26d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupService.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -public interface BackupService -{ - boolean isBackupAvailable(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupServiceManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupServiceManager.java deleted file mode 100644 index 7eef51dbd7f0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupServiceManager.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.inject.Inject; - -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public class BackupServiceManager - implements BackupService -{ - private final Optional backupStore; - - @Inject - public BackupServiceManager(Optional backupStore) - { - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - } - - @Override - public boolean isBackupAvailable() - { - return backupStore.isPresent(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupStore.java deleted file mode 100644 index dff2ab3637be..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/BackupStore.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import java.io.File; -import java.util.UUID; - -public interface BackupStore -{ - /** - * Backup a shard by copying it to the backup store. - * - * @param uuid shard UUID - * @param source the source file - */ - void backupShard(UUID uuid, File source); - - /** - * Restore a shard by copying it from the backup store. - * - * @param uuid shard UUID - * @param target the destination file - */ - void restoreShard(UUID uuid, File target); - - /** - * Delete shard from the backup store if it exists. - * - * @param uuid shard UUID - * @return {@code true} if the shard was deleted; {@code false} if it did not exist - */ - boolean deleteShard(UUID uuid); - - /** - * Check if a shard exists in the backup store. - * - * @param uuid shard UUID - * @return if the shard exists - */ - boolean shardExists(UUID uuid); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupConfig.java deleted file mode 100644 index 7f54dc56ce44..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import jakarta.validation.constraints.NotNull; - -import java.io.File; - -public class FileBackupConfig -{ - private File backupDirectory; - - @NotNull - public File getBackupDirectory() - { - return backupDirectory; - } - - @Config("backup.directory") - @ConfigDescription("Base directory to use for the backup copy of shard data") - public FileBackupConfig setBackupDirectory(File backupDirectory) - { - this.backupDirectory = backupDirectory; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupModule.java deleted file mode 100644 index 604574c1605c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupModule.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Scopes; - -import static io.airlift.configuration.ConfigBinder.configBinder; - -public class FileBackupModule - implements Module -{ - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(FileBackupConfig.class); - binder.bind(BackupStore.class).to(FileBackupStore.class).in(Scopes.SINGLETON); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupStore.java deleted file mode 100644 index 33943038ea91..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/FileBackupStore.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; -import io.trino.spi.TrinoException; -import jakarta.annotation.PostConstruct; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.UUID; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_NOT_FOUND; -import static io.trino.plugin.raptor.legacy.storage.FileStorageService.getFileSystemPath; -import static java.nio.file.Files.deleteIfExists; -import static java.util.Objects.requireNonNull; - -public class FileBackupStore - implements BackupStore -{ - private final File baseDir; - - @Inject - public FileBackupStore(FileBackupConfig config) - { - this(config.getBackupDirectory()); - } - - public FileBackupStore(File baseDir) - { - this.baseDir = requireNonNull(baseDir, "baseDir is null"); - } - - @PostConstruct - public void start() - { - createDirectories(baseDir); - } - - @Override - public void backupShard(UUID uuid, File source) - { - File backupFile = getBackupFile(uuid); - - try { - try { - // Optimistically assume the file can be created - copyFile(source, backupFile); - } - catch (FileNotFoundException e) { - createDirectories(backupFile.getParentFile()); - copyFile(source, backupFile); - } - } - catch (IOException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to create backup shard file", e); - } - } - - @Override - public void restoreShard(UUID uuid, File target) - { - try { - copyFile(getBackupFile(uuid), target); - } - catch (FileNotFoundException e) { - throw new TrinoException(RAPTOR_BACKUP_NOT_FOUND, "Backup shard not found: " + uuid, e); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to copy backup shard: " + uuid, e); - } - } - - @Override - public boolean deleteShard(UUID uuid) - { - try { - return deleteIfExists(getBackupFile(uuid).toPath()); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to delete backup shard: " + uuid, e); - } - } - - @Override - public boolean shardExists(UUID uuid) - { - return getBackupFile(uuid).isFile(); - } - - @VisibleForTesting - public File getBackupFile(UUID uuid) - { - return getFileSystemPath(baseDir, uuid); - } - - private static void createDirectories(File dir) - { - if (!dir.mkdirs() && !dir.isDirectory()) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed creating directories: " + dir); - } - } - - private static void copyFile(File source, File target) - throws IOException - { - try (InputStream in = new FileInputStream(source); - FileOutputStream out = new FileOutputStream(target)) { - in.transferTo(out); - out.flush(); - out.getFD().sync(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ForHttpBackup.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ForHttpBackup.java deleted file mode 100644 index 7f62f6a0a2b3..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ForHttpBackup.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.inject.BindingAnnotation; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -@Retention(RUNTIME) -@Target({FIELD, PARAMETER, METHOD}) -@BindingAnnotation -public @interface ForHttpBackup -{ -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupConfig.java deleted file mode 100644 index 2dfceae6015a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import jakarta.validation.constraints.NotNull; - -import java.net.URI; - -public class HttpBackupConfig -{ - private URI uri; - - @NotNull - public URI getUri() - { - return uri; - } - - @Config("backup.http.uri") - @ConfigDescription("Backup service base URI") - public HttpBackupConfig setUri(URI uri) - { - this.uri = uri; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupModule.java deleted file mode 100644 index 739bc9fa815b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupModule.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; - -import java.net.URI; -import java.util.function.Supplier; - -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.http.client.HttpClientBinder.httpClientBinder; - -public class HttpBackupModule - implements Module -{ - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(HttpBackupConfig.class); - binder.bind(BackupStore.class).to(HttpBackupStore.class).in(Scopes.SINGLETON); - - httpClientBinder(binder).bindHttpClient("backup", ForHttpBackup.class); - } - - @Provides - @Singleton - @ForHttpBackup - public Supplier createBackupUriSupplier(HttpBackupConfig config) - { - URI uri = config.getUri(); - return () -> uri; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupStore.java deleted file mode 100644 index cac0405f8869..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/HttpBackupStore.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.io.ByteStreams; -import com.google.inject.Inject; -import io.airlift.http.client.FileBodyGenerator; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; -import io.airlift.http.client.StatusResponseHandler.StatusResponse; -import io.airlift.slice.XxHash64; -import io.trino.spi.NodeManager; -import io.trino.spi.TrinoException; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.UUID; -import java.util.function.Supplier; - -import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static com.google.common.net.MediaType.APPLICATION_BINARY; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.Request.Builder.prepareHead; -import static io.airlift.http.client.Request.Builder.preparePut; -import static io.airlift.http.client.ResponseHandlerUtils.propagate; -import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_ERROR; -import static java.lang.String.format; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.requireNonNull; - -public class HttpBackupStore - implements BackupStore -{ - public static final String TRINO_ENVIRONMENT = "X-Trino-Environment"; - public static final String CONTENT_XXH64 = "X-Content-XXH64"; - - private final HttpClient httpClient; - private final Supplier baseUriSupplier; - private final String environment; - - @Inject - public HttpBackupStore( - @ForHttpBackup HttpClient httpClient, - @ForHttpBackup Supplier baseUriSupplier, - NodeManager nodeManager) - { - this.httpClient = requireNonNull(httpClient, "httpClient is null"); - this.baseUriSupplier = requireNonNull(baseUriSupplier, "baseUriSupplier is null"); - this.environment = nodeManager.getEnvironment(); - } - - @Override - public void backupShard(UUID uuid, File source) - { - Request request = preparePut() - .addHeader(TRINO_ENVIRONMENT, environment) - .addHeader(CONTENT_TYPE, APPLICATION_BINARY.toString()) - .addHeader(CONTENT_XXH64, format("%016x", xxHash64(source))) - .setUri(shardUri(uuid)) - .setBodyGenerator(new FileBodyGenerator(source.toPath())) - .build(); - - try { - StatusResponse status = httpClient.execute(request, createStatusResponseHandler()); - if (!isOk(status)) { - throw badResponse(status); - } - } - catch (RuntimeException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to backup shard: " + uuid, e); - } - } - - @Override - public void restoreShard(UUID uuid, File target) - { - Request request = prepareGet() - .addHeader(TRINO_ENVIRONMENT, environment) - .setUri(shardUri(uuid)) - .build(); - - try { - StatusResponse status = httpClient.execute(request, new FileResponseHandler(target)); - if (isNotFound(status) || isGone(status)) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Backup shard not found: " + uuid); - } - if (!isOk(status)) { - throw badResponse(status); - } - } - catch (IOException | RuntimeException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to restore shard: " + uuid, e); - } - } - - @Override - public boolean deleteShard(UUID uuid) - { - Request request = prepareDelete() - .addHeader(TRINO_ENVIRONMENT, environment) - .setUri(shardUri(uuid)) - .build(); - - try { - StatusResponse status = httpClient.execute(request, createStatusResponseHandler()); - if (isOk(status) || isGone(status)) { - return true; - } - if (isNotFound(status)) { - return false; - } - throw badResponse(status); - } - catch (RuntimeException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to delete shard: " + uuid, e); - } - } - - @Override - public boolean shardExists(UUID uuid) - { - Request request = prepareHead() - .addHeader(TRINO_ENVIRONMENT, environment) - .setUri(shardUri(uuid)) - .build(); - - try { - StatusResponse status = httpClient.execute(request, createStatusResponseHandler()); - if (isOk(status)) { - return true; - } - if (isNotFound(status) || isGone(status)) { - return false; - } - throw badResponse(status); - } - catch (RuntimeException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to check if shard exists: " + uuid, e); - } - } - - private URI shardUri(UUID uuid) - { - return uriBuilderFrom(baseUriSupplier.get()) - .appendPath(uuid.toString().toLowerCase(ENGLISH)) - .build(); - } - - private static boolean isOk(StatusResponse response) - { - return (response.getStatusCode() == HttpStatus.OK.code()) || - (response.getStatusCode() == HttpStatus.NO_CONTENT.code()); - } - - private static boolean isNotFound(StatusResponse response) - { - return response.getStatusCode() == HttpStatus.NOT_FOUND.code(); - } - - private static boolean isGone(StatusResponse response) - { - return response.getStatusCode() == HttpStatus.GONE.code(); - } - - private static RuntimeException badResponse(StatusResponse response) - { - throw new RuntimeException("Request failed with HTTP status " + response.getStatusCode()); - } - - private static long xxHash64(File file) - { - try (InputStream in = new FileInputStream(file)) { - return XxHash64.hash(in); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Failed to read file: " + file, e); - } - } - - private static class FileResponseHandler - implements ResponseHandler - { - private final File file; - - private FileResponseHandler(File file) - { - this.file = requireNonNull(file, "file is null"); - } - - @Override - public StatusResponse handleException(Request request, Exception exception) - { - throw propagate(request, exception); - } - - @Override - public StatusResponse handle(Request request, Response response) - throws IOException - { - StatusResponse status = createStatusResponse(response); - if (isOk(status)) { - writeFile(response.getInputStream()); - } - return status; - } - - private void writeFile(InputStream in) - throws IOException - { - try (FileOutputStream out = new FileOutputStream(file)) { - ByteStreams.copy(in, out); - out.flush(); - out.getFD().sync(); - } - } - - private static StatusResponse createStatusResponse(Response response) - { - return new StatusResponse(response.getStatusCode(), response.getHeaders()); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ManagedBackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ManagedBackupStore.java deleted file mode 100644 index 45b31e75681c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/ManagedBackupStore.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.airlift.log.Logger; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.io.File; -import java.util.UUID; - -import static java.util.Objects.requireNonNull; - -public class ManagedBackupStore - implements BackupStore -{ - private final BackupStore store; - private final Logger log; - - private final BackupOperationStats backupShard = new BackupOperationStats(); - private final BackupOperationStats restoreShard = new BackupOperationStats(); - private final BackupOperationStats deleteShard = new BackupOperationStats(); - private final BackupOperationStats shardExists = new BackupOperationStats(); - - public ManagedBackupStore(BackupStore store) - { - this.store = requireNonNull(store, "store is null"); - this.log = Logger.get(store.getClass()); - } - - @Override - public void backupShard(UUID uuid, File source) - { - log.debug("Creating shard backup: %s", uuid); - backupShard.run(() -> store.backupShard(uuid, source)); - } - - @Override - public void restoreShard(UUID uuid, File target) - { - log.debug("Restoring shard backup: %s", uuid); - restoreShard.run(() -> store.restoreShard(uuid, target)); - } - - @Override - public boolean deleteShard(UUID uuid) - { - log.debug("Deleting shard backup: %s", uuid); - return deleteShard.run(() -> store.deleteShard(uuid)); - } - - @Override - public boolean shardExists(UUID uuid) - { - return shardExists.run(() -> store.shardExists(uuid)); - } - - @Managed - @Nested - public BackupOperationStats getBackupShard() - { - return backupShard; - } - - @Managed - @Nested - public BackupOperationStats getRestoreShard() - { - return restoreShard; - } - - @Managed - @Nested - public BackupOperationStats getDeleteShard() - { - return deleteShard; - } - - @Managed - @Nested - public BackupOperationStats getShardExists() - { - return shardExists; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/TimeoutBackupStore.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/TimeoutBackupStore.java deleted file mode 100644 index f627b6bf369e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/backup/TimeoutBackupStore.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.util.concurrent.SimpleTimeLimiter; -import com.google.common.util.concurrent.TimeLimiter; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.concurrent.ExecutorServiceAdapter; -import io.airlift.units.Duration; -import io.trino.spi.TrinoException; -import jakarta.annotation.PreDestroy; - -import java.io.File; -import java.util.UUID; -import java.util.concurrent.ExecutorService; - -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_TIMEOUT; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class TimeoutBackupStore - implements BackupStore -{ - private final ExecutorService executor; - private final BackupStore store; - - public TimeoutBackupStore(BackupStore store, String connectorId, Duration timeout, int maxThreads) - { - requireNonNull(store, "store is null"); - requireNonNull(connectorId, "connectorId is null"); - requireNonNull(timeout, "timeout is null"); - - this.executor = newCachedThreadPool(daemonThreadsNamed("backup-proxy-" + connectorId + "-%s")); - this.store = timeLimited(store, BackupStore.class, timeout, executor, maxThreads); - } - - @PreDestroy - public void shutdown() - { - executor.shutdownNow(); - } - - @Override - public void backupShard(UUID uuid, File source) - { - try { - store.backupShard(uuid, source); - } - catch (UncheckedTimeoutException e) { - timeoutException(uuid, "Shard backup timed out"); - } - } - - @Override - public void restoreShard(UUID uuid, File target) - { - try { - store.restoreShard(uuid, target); - } - catch (UncheckedTimeoutException e) { - timeoutException(uuid, "Shard restore timed out"); - } - } - - @Override - public boolean deleteShard(UUID uuid) - { - try { - return store.deleteShard(uuid); - } - catch (UncheckedTimeoutException e) { - throw timeoutException(uuid, "Shard delete timed out"); - } - } - - @Override - public boolean shardExists(UUID uuid) - { - try { - return store.shardExists(uuid); - } - catch (UncheckedTimeoutException e) { - throw timeoutException(uuid, "Shard existence check timed out"); - } - } - - private static T timeLimited(T target, Class clazz, Duration timeout, ExecutorService executor, int maxThreads) - { - executor = new ExecutorServiceAdapter(new BoundedExecutor(executor, maxThreads)); - TimeLimiter limiter = SimpleTimeLimiter.create(executor); - return limiter.newProxy(target, clazz, timeout.toMillis(), MILLISECONDS); - } - - private static TrinoException timeoutException(UUID uuid, String message) - { - throw new TrinoException(RAPTOR_BACKUP_TIMEOUT, message + ": " + uuid); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/AssignmentLimiter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/AssignmentLimiter.java deleted file mode 100644 index fedc32a774c2..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/AssignmentLimiter.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.base.Ticker; -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.spi.Node; -import io.trino.spi.TrinoException; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.OptionalLong; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; - -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_REASSIGNMENT_DELAY; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_REASSIGNMENT_THROTTLE; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toSet; - -public class AssignmentLimiter -{ - private static final Logger log = Logger.get(AssignmentLimiter.class); - - private final NodeSupplier nodeSupplier; - private final Ticker ticker; - private final Duration reassignmentDelay; - private final Duration reassignmentInterval; - - private final ScheduledExecutorService scheduler = newScheduledThreadPool(1, daemonThreadsNamed("assignment-limiter")); - private final AtomicBoolean started = new AtomicBoolean(); - - @GuardedBy("this") - private final Map delayedNodes = new HashMap<>(); - @GuardedBy("this") - private final Set offlineNodes = new HashSet<>(); - @GuardedBy("this") - private OptionalLong lastOfflined = OptionalLong.empty(); - - @Inject - public AssignmentLimiter(NodeSupplier nodeSupplier, Ticker ticker, MetadataConfig config) - { - this(nodeSupplier, ticker, config.getReassignmentDelay(), config.getReassignmentInterval()); - } - - public AssignmentLimiter(NodeSupplier nodeSupplier, Ticker ticker, Duration reassignmentDelay, Duration reassignmentInterval) - { - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.ticker = requireNonNull(ticker, "ticker is null"); - this.reassignmentDelay = requireNonNull(reassignmentDelay, "reassignmentDelay is null"); - this.reassignmentInterval = requireNonNull(reassignmentInterval, "reassignmentInterval is null"); - } - - @PostConstruct - public void start() - { - if (!started.getAndSet(true)) { - scheduler.scheduleWithFixedDelay(() -> { - try { - clearOnlineNodes(); - } - catch (Throwable t) { - log.error(t, "Error clearing online nodes"); - } - }, 2, 2, SECONDS); - } - } - - @PreDestroy - public void shutdown() - { - scheduler.shutdownNow(); - } - - public synchronized void checkAssignFrom(String nodeIdentifier) - { - if (offlineNodes.contains(nodeIdentifier)) { - return; - } - - long now = ticker.read(); - long start = delayedNodes.computeIfAbsent(nodeIdentifier, key -> now); - Duration delay = new Duration(now - start, NANOSECONDS); - - if (delay.compareTo(reassignmentDelay) < 0) { - throw new TrinoException(RAPTOR_REASSIGNMENT_DELAY, format( - "Reassignment delay is in effect for node %s (elapsed: %s)", - nodeIdentifier, - delay.convertToMostSuccinctTimeUnit())); - } - - if (lastOfflined.isPresent()) { - delay = new Duration(now - lastOfflined.getAsLong(), NANOSECONDS); - if (delay.compareTo(reassignmentInterval) < 0) { - throw new TrinoException(RAPTOR_REASSIGNMENT_THROTTLE, format( - "Reassignment throttle is in effect for node %s (elapsed: %s)", - nodeIdentifier, - delay.convertToMostSuccinctTimeUnit())); - } - } - - delayedNodes.remove(nodeIdentifier); - offlineNodes.add(nodeIdentifier); - lastOfflined = OptionalLong.of(now); - } - - private void clearOnlineNodes() - { - Set onlineNodes = nodeSupplier.getWorkerNodes().stream() - .map(Node::getNodeIdentifier) - .collect(toSet()); - - synchronized (this) { - delayedNodes.keySet().removeAll(onlineNodes); - offlineNodes.removeAll(onlineNodes); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketNode.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketNode.java deleted file mode 100644 index 0d329b0b9f37..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketNode.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import java.util.Objects; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class BucketNode -{ - private final int bucketNumber; - private final String nodeIdentifier; - - public BucketNode(int bucketNumber, String nodeIdentifier) - { - checkArgument(bucketNumber >= 0, "bucket number must be positive"); - this.bucketNumber = bucketNumber; - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - } - - public int getBucketNumber() - { - return bucketNumber; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if ((o == null) || (getClass() != o.getClass())) { - return false; - } - BucketNode that = (BucketNode) o; - return (bucketNumber == that.bucketNumber) && - Objects.equals(nodeIdentifier, that.nodeIdentifier); - } - - @Override - public int hashCode() - { - return Objects.hash(bucketNumber, nodeIdentifier); - } - - @Override - public String toString() - { - return bucketNumber + ":" + nodeIdentifier; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketReassigner.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketReassigner.java deleted file mode 100644 index 680c40a8c112..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketReassigner.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ThreadLocalRandom; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class BucketReassigner -{ - private final Map nodeBucketCounts = new HashMap<>(); - private final List activeNodes; - private final List bucketNodes; - private boolean initialized; - - public BucketReassigner(Set activeNodes, List bucketNodes) - { - checkArgument(!activeNodes.isEmpty(), "activeNodes must not be empty"); - this.activeNodes = ImmutableList.copyOf(requireNonNull(activeNodes, "activeNodes is null")); - this.bucketNodes = requireNonNull(bucketNodes, "bucketNodes is null"); - } - - // NOTE: This method is not thread safe - public String getNextReassignmentDestination() - { - if (!initialized) { - for (String node : activeNodes) { - nodeBucketCounts.put(node, 0); - } - for (BucketNode bucketNode : bucketNodes) { - nodeBucketCounts.computeIfPresent(bucketNode.getNodeIdentifier(), (node, bucketCount) -> bucketCount + 1); - } - initialized = true; - } - - String assignedNode; - if (activeNodes.size() > 1) { - assignedNode = randomTwoChoices(); - } - else { - assignedNode = activeNodes.get(0); - } - - nodeBucketCounts.compute(assignedNode, (node, count) -> count + 1); - return assignedNode; - } - - private String randomTwoChoices() - { - // Purely random choices can overload unlucky node while selecting the least loaded one based on stale - // local information can overload the previous idle node. Here we randomly pick 2 nodes and select the - // less loaded one. This prevents those issues and renders good enough load balance. - int randomPosition = ThreadLocalRandom.current().nextInt(activeNodes.size()); - int randomOffset = ThreadLocalRandom.current().nextInt(1, activeNodes.size()); - String candidate1 = activeNodes.get(randomPosition); - String candidate2 = activeNodes.get((randomPosition + randomOffset) % activeNodes.size()); - - return (nodeBucketCounts.get(candidate1) <= nodeBucketCounts.get(candidate2)) ? candidate1 : candidate2; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketShards.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketShards.java deleted file mode 100644 index d38f1efe0424..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/BucketShards.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableSet; - -import java.util.Objects; -import java.util.OptionalInt; -import java.util.Set; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class BucketShards -{ - private final OptionalInt bucketNumber; - private final Set shards; - - public BucketShards(OptionalInt bucketNumber, Set shards) - { - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.shards = ImmutableSet.copyOf(requireNonNull(shards, "shards is null")); - } - - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - public Set getShards() - { - return shards; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - BucketShards other = (BucketShards) obj; - return Objects.equals(this.bucketNumber, other.bucketNumber) && - Objects.equals(this.shards, other.shards); - } - - @Override - public int hashCode() - { - return Objects.hash(bucketNumber, shards); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .add("shards", shards) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnInfo.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnInfo.java deleted file mode 100644 index d4f6b36cce8f..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnInfo.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.type.Type; - -import static java.util.Objects.requireNonNull; - -public class ColumnInfo -{ - private final long columnId; - private final Type type; - - public ColumnInfo(long columnId, Type type) - { - this.columnId = columnId; - this.type = requireNonNull(type, "type is null"); - } - - public long getColumnId() - { - return columnId; - } - - public Type getType() - { - return type; - } - - @Override - public String toString() - { - return columnId + ":" + type; - } - - public static ColumnInfo fromHandle(RaptorColumnHandle handle) - { - return new ColumnInfo(handle.getColumnId(), handle.getColumnType()); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnMetadataRow.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnMetadataRow.java deleted file mode 100644 index 0d5077f19b8f..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnMetadataRow.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import java.util.OptionalInt; - -import static java.util.Objects.requireNonNull; - -public class ColumnMetadataRow -{ - private final long tableId; - private final long columnId; - private final String columnName; - private final OptionalInt sortOrdinalPosition; - private final OptionalInt bucketOrdinalPosition; - - public ColumnMetadataRow(long tableId, long columnId, String columnName, OptionalInt sortOrdinalPosition, OptionalInt bucketOrdinalPosition) - { - this.tableId = tableId; - this.columnId = columnId; - this.columnName = requireNonNull(columnName, "columnName is null"); - this.sortOrdinalPosition = requireNonNull(sortOrdinalPosition, "sortOrdinalPosition is null"); - this.bucketOrdinalPosition = requireNonNull(bucketOrdinalPosition, "bucketOrdinalPosition is null"); - } - - public long getTableId() - { - return tableId; - } - - public long getColumnId() - { - return columnId; - } - - public String getColumnName() - { - return columnName; - } - - public OptionalInt getSortOrdinalPosition() - { - return sortOrdinalPosition; - } - - public OptionalInt getBucketOrdinalPosition() - { - return bucketOrdinalPosition; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnStats.java deleted file mode 100644 index 77b63ffbfe4d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ColumnStats.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.annotation.Nullable; - -import static com.google.common.base.MoreObjects.toStringHelper; - -public class ColumnStats -{ - private final long columnId; - private final Object min; - private final Object max; - - @JsonCreator - public ColumnStats( - @JsonProperty("columnId") long columnId, - @JsonProperty("min") @Nullable Object min, - @JsonProperty("max") @Nullable Object max) - { - this.columnId = columnId; - this.min = min; - this.max = max; - } - - @JsonProperty - public long getColumnId() - { - return columnId; - } - - @Nullable - @JsonProperty - public Object getMin() - { - return min; - } - - @Nullable - @JsonProperty - public Object getMax() - { - return max; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("columnId", columnId) - .add("min", min) - .add("max", max) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseConfig.java deleted file mode 100644 index 26befd10a601..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import jakarta.validation.constraints.NotNull; - -public class DatabaseConfig -{ - private String databaseType; - - @NotNull - public String getDatabaseType() - { - return databaseType; - } - - @Config("metadata.db.type") - @ConfigDescription("Metadata database type (supported types: h2, mysql)") - public DatabaseConfig setDatabaseType(String databaseType) - { - this.databaseType = databaseType; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseMetadataModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseMetadataModule.java deleted file mode 100644 index 4cf735884e1d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseMetadataModule.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import org.jdbi.v3.core.ConnectionFactory; -import org.jdbi.v3.core.Jdbi; - -import java.sql.DriverManager; - -import static io.airlift.configuration.ConditionalModule.conditionalModule; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static java.lang.String.format; - -public class DatabaseMetadataModule - extends AbstractConfigurationAwareModule -{ - @Override - protected void setup(Binder ignored) - { - install(conditionalModule( - DatabaseConfig.class, - config -> "mysql".equals(config.getDatabaseType()), - new MySqlModule())); - - install(conditionalModule( - DatabaseConfig.class, - config -> "h2".equals(config.getDatabaseType()), - new H2Module())); - } - - private static class MySqlModule - implements Module - { - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(JdbcDatabaseConfig.class); - } - - @Provides - @Singleton - @ForMetadata - public static ConnectionFactory createConnectionFactory(JdbcDatabaseConfig config) - { - String url = config.getUrl(); - return () -> DriverManager.getConnection(url); - } - - @Provides - @Singleton - public static DaoSupplier createShardDaoSupplier(@ForMetadata Jdbi dbi) - { - return new DaoSupplier<>(dbi, MySqlShardDao.class); - } - } - - private static class H2Module - implements Module - { - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(H2DatabaseConfig.class); - } - - @Provides - @Singleton - @ForMetadata - public static ConnectionFactory createConnectionFactory(H2DatabaseConfig config) - { - String url = format("jdbc:h2:%s;DB_CLOSE_DELAY=-1", config.getFilename()); - return () -> DriverManager.getConnection(url); - } - - @Provides - @Singleton - public static DaoSupplier createShardDaoSupplier(@ForMetadata Jdbi dbi) - { - return new DaoSupplier<>(dbi, H2ShardDao.class); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardManager.java deleted file mode 100644 index 447981fcf0e0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardManager.java +++ /dev/null @@ -1,933 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.base.Joiner; -import com.google.common.base.Ticker; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.UncheckedExecutionException; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.cache.NonEvictableLoadingCache; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerDao; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.Node; -import io.trino.spi.TrinoException; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import org.h2.jdbc.JdbcConnection; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.HandleConsumer; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.JdbiException; -import org.jdbi.v3.core.result.ResultIterator; - -import java.sql.Connection; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.StringJoiner; -import java.util.UUID; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Throwables.throwIfInstanceOf; -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.Iterables.partition; -import static io.trino.cache.SafeCaches.buildNonEvictableCache; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS; -import static io.trino.plugin.raptor.legacy.storage.ColumnIndexStatsUtils.jdbcType; -import static io.trino.plugin.raptor.legacy.storage.ShardStats.MAX_BINARY_INDEX_SIZE; -import static io.trino.plugin.raptor.legacy.util.ArrayUtil.intArrayFromBytes; -import static io.trino.plugin.raptor.legacy.util.ArrayUtil.intArrayToBytes; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.bindOptionalInt; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.isSyntaxOrAccessError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.isTransactionCacheFullError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runIgnoringConstraintViolation; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runTransaction; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidToBytes; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static io.trino.spi.StandardErrorCode.NO_NODES_AVAILABLE; -import static io.trino.spi.StandardErrorCode.SERVER_STARTING_UP; -import static io.trino.spi.StandardErrorCode.TRANSACTION_CONFLICT; -import static java.lang.Boolean.TRUE; -import static java.lang.Math.multiplyExact; -import static java.lang.String.format; -import static java.sql.Statement.RETURN_GENERATED_KEYS; -import static java.util.Arrays.asList; -import static java.util.Collections.nCopies; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - -public class DatabaseShardManager - implements ShardManager -{ - private static final Logger log = Logger.get(DatabaseShardManager.class); - - private static final String INDEX_TABLE_PREFIX = "x_shards_t"; - private static final int MAX_ADD_COLUMN_ATTEMPTS = 100; - - private final Jdbi dbi; - private final DaoSupplier shardDaoSupplier; - private final ShardDao dao; - private final NodeSupplier nodeSupplier; - private final AssignmentLimiter assignmentLimiter; - private final Ticker ticker; - private final Duration startupGracePeriod; - private final long startTime; - - private final NonEvictableLoadingCache nodeIdCache = buildNonEvictableCache( - CacheBuilder.newBuilder().maximumSize(10_000), - CacheLoader.from(this::loadNodeId)); - - private final NonEvictableLoadingCache> bucketAssignmentsCache = buildNonEvictableCache( - CacheBuilder.newBuilder().expireAfterWrite(1, SECONDS), - CacheLoader.from(this::loadBucketAssignments)); - - @Inject - public DatabaseShardManager( - @ForMetadata Jdbi dbi, - DaoSupplier shardDaoSupplier, - NodeSupplier nodeSupplier, - AssignmentLimiter assignmentLimiter, - Ticker ticker, - MetadataConfig config) - { - this(dbi, shardDaoSupplier, nodeSupplier, assignmentLimiter, ticker, config.getStartupGracePeriod()); - } - - public DatabaseShardManager( - Jdbi dbi, - DaoSupplier shardDaoSupplier, - NodeSupplier nodeSupplier, - AssignmentLimiter assignmentLimiter, - Ticker ticker, - Duration startupGracePeriod) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.shardDaoSupplier = requireNonNull(shardDaoSupplier, "shardDaoSupplier is null"); - this.dao = shardDaoSupplier.onDemand(); - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.assignmentLimiter = requireNonNull(assignmentLimiter, "assignmentLimiter is null"); - this.ticker = requireNonNull(ticker, "ticker is null"); - this.startupGracePeriod = requireNonNull(startupGracePeriod, "startupGracePeriod is null"); - this.startTime = ticker.read(); - } - - @Override - public void createTable(long tableId, List columns, boolean bucketed, OptionalLong temporalColumnId) - { - StringJoiner tableColumns = new StringJoiner(",\n ", " ", ",\n").setEmptyValue(""); - - for (ColumnInfo column : columns) { - String columnType = sqlColumnType(column.getType()); - if (columnType != null) { - tableColumns.add(minColumn(column.getColumnId()) + " " + columnType); - tableColumns.add(maxColumn(column.getColumnId()) + " " + columnType); - } - } - - StringJoiner coveringIndexColumns = new StringJoiner(", "); - - // Add the max temporal column first to accelerate queries that usually scan recent data - temporalColumnId.ifPresent(id -> coveringIndexColumns.add(maxColumn(id))); - temporalColumnId.ifPresent(id -> coveringIndexColumns.add(minColumn(id))); - - String sql; - if (bucketed) { - coveringIndexColumns - .add("bucket_number") - .add("shard_id") - .add("shard_uuid"); - - sql = "" + - "CREATE TABLE " + shardIndexTable(tableId) + " (\n" + - " shard_id BIGINT NOT NULL,\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " bucket_number INT NOT NULL\n," + - tableColumns + - " PRIMARY KEY (bucket_number, shard_uuid),\n" + - " UNIQUE (shard_id),\n" + - " UNIQUE (shard_uuid),\n" + - " UNIQUE (" + coveringIndexColumns + ")\n" + - ")"; - } - else { - coveringIndexColumns - .add("node_ids") - .add("shard_id") - .add("shard_uuid"); - - sql = "" + - "CREATE TABLE " + shardIndexTable(tableId) + " (\n" + - " shard_id BIGINT NOT NULL,\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " node_ids VARBINARY(128) NOT NULL,\n" + - tableColumns + - " PRIMARY KEY (node_ids, shard_uuid),\n" + - " UNIQUE (shard_id),\n" + - " UNIQUE (shard_uuid),\n" + - " UNIQUE (" + coveringIndexColumns + ")\n" + - ")"; - } - - try (Handle handle = dbi.open()) { - handle.execute(sql); - } - catch (JdbiException e) { - throw metadataError(e); - } - } - - @Override - public void dropTable(long tableId) - { - runTransaction(dbi, handle -> { - lockTable(handle, tableId); - - ShardDao shardDao = shardDaoSupplier.attach(handle); - shardDao.insertDeletedShards(tableId); - shardDao.dropShardNodes(tableId); - shardDao.dropShards(tableId); - - handle.attach(ShardOrganizerDao.class).dropOrganizerJobs(tableId); - - MetadataDao dao = handle.attach(MetadataDao.class); - dao.dropColumns(tableId); - dao.dropTable(tableId); - return null; - }); - - // TODO: add a cleanup process for leftover index tables - // It is not possible to drop the index tables in a transaction. - try (Handle handle = dbi.open()) { - handle.execute("DROP TABLE " + shardIndexTable(tableId)); - } - catch (JdbiException e) { - log.warn(e, "Failed to drop index table %s", shardIndexTable(tableId)); - } - } - - @Override - public void addColumn(long tableId, ColumnInfo column) - { - String columnType = sqlColumnType(column.getType()); - if (columnType == null) { - // TODO we should probably fail here - return; - } - - String sql = format("ALTER TABLE %s ADD COLUMN (%s %s, %s %s)", - shardIndexTable(tableId), - minColumn(column.getColumnId()), columnType, - maxColumn(column.getColumnId()), columnType); - - int attempts = 0; - while (true) { - attempts++; - try (Handle handle = dbi.open()) { - handle.execute(sql); - } - catch (JdbiException e) { - if (isSyntaxOrAccessError(e)) { - // exit when column already exists - return; - } - if (attempts >= MAX_ADD_COLUMN_ATTEMPTS) { - throw metadataError(e); - } - log.warn(e, "Failed to alter table on attempt %s, will retry. SQL: %s", attempts, sql); - try { - SECONDS.sleep(3); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw metadataError(ie); - } - } - } - } - - @Override - public void commitShards(long transactionId, long tableId, List columns, Collection shards, Optional externalBatchId, long updateTime) - { - // attempt to fail up front with a proper exception - if (externalBatchId.isPresent() && dao.externalBatchExists(externalBatchId.get())) { - throw new TrinoException(RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS, "External batch already exists: " + externalBatchId.get()); - } - - Map nodeIds = toNodeIdMap(shards); - - runCommit(transactionId, handle -> { - externalBatchId.ifPresent(shardDaoSupplier.attach(handle)::insertExternalBatch); - lockTable(handle, tableId); - insertShardsAndIndex(tableId, columns, shards, nodeIds, handle); - - ShardStats stats = shardStats(shards); - MetadataDao metadata = handle.attach(MetadataDao.class); - metadata.updateTableStats(tableId, shards.size(), stats.getRowCount(), stats.getCompressedSize(), stats.getUncompressedSize()); - metadata.updateTableVersion(tableId, updateTime); - }); - } - - @Override - public void replaceShardUuids(long transactionId, long tableId, List columns, Set oldShardUuids, Collection newShards, OptionalLong updateTime) - { - Map nodeIds = toNodeIdMap(newShards); - - runCommit(transactionId, handle -> { - lockTable(handle, tableId); - - if (updateTime.isEmpty() && handle.attach(MetadataDao.class).isMaintenanceBlockedLocked(tableId)) { - throw new TrinoException(TRANSACTION_CONFLICT, "Maintenance is blocked for table"); - } - - ShardStats newStats = shardStats(newShards); - long rowCount = newStats.getRowCount(); - long compressedSize = newStats.getCompressedSize(); - long uncompressedSize = newStats.getUncompressedSize(); - - for (List shards : partition(newShards, 1000)) { - insertShardsAndIndex(tableId, columns, shards, nodeIds, handle); - } - - for (List uuids : partition(oldShardUuids, 1000)) { - ShardStats stats = deleteShardsAndIndex(tableId, ImmutableSet.copyOf(uuids), handle); - rowCount -= stats.getRowCount(); - compressedSize -= stats.getCompressedSize(); - uncompressedSize -= stats.getUncompressedSize(); - } - - long shardCount = newShards.size() - oldShardUuids.size(); - - if (!oldShardUuids.isEmpty() || !newShards.isEmpty()) { - MetadataDao metadata = handle.attach(MetadataDao.class); - metadata.updateTableStats(tableId, shardCount, rowCount, compressedSize, uncompressedSize); - updateTime.ifPresent(time -> metadata.updateTableVersion(tableId, time)); - } - }); - } - - private void runCommit(long transactionId, HandleConsumer callback) - { - int maxAttempts = 5; - for (int attempt = 1; attempt <= maxAttempts; attempt++) { - try { - dbi.useTransaction(handle -> { - ShardDao dao = shardDaoSupplier.attach(handle); - if (commitTransaction(dao, transactionId)) { - callback.useHandle(handle); - dao.deleteCreatedShards(transactionId); - } - }); - return; - } - catch (JdbiException | SQLException e) { - if (isTransactionCacheFullError(e)) { - throw metadataError(e, "Transaction too large"); - } - - if (e.getCause() != null) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - } - - if (attempt == maxAttempts) { - throw metadataError(e); - } - log.warn(e, "Failed to commit shards on attempt %d, will retry.", attempt); - try { - SECONDS.sleep(multiplyExact(attempt, 2)); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw metadataError(ie); - } - } - } - } - - private static boolean commitTransaction(ShardDao dao, long transactionId) - { - if (dao.finalizeTransaction(transactionId, true) != 1) { - if (TRUE.equals(dao.transactionSuccessful(transactionId))) { - return false; - } - throw new TrinoException(TRANSACTION_CONFLICT, "Transaction commit failed. Please retry the operation."); - } - return true; - } - - private ShardStats deleteShardsAndIndex(long tableId, Set shardUuids, Handle handle) - throws SQLException - { - String args = Joiner.on(",").join(nCopies(shardUuids.size(), "?")); - - ImmutableSet.Builder shardIdSet = ImmutableSet.builder(); - long rowCount = 0; - long compressedSize = 0; - long uncompressedSize = 0; - - String selectShards = format("" + - "SELECT shard_id, row_count, compressed_size, uncompressed_size\n" + - "FROM shards\n" + - "WHERE shard_uuid IN (%s)", args); - - try (PreparedStatement statement = handle.getConnection().prepareStatement(selectShards)) { - bindUuids(statement, shardUuids); - try (ResultSet rs = statement.executeQuery()) { - while (rs.next()) { - shardIdSet.add(rs.getLong("shard_id")); - rowCount += rs.getLong("row_count"); - compressedSize += rs.getLong("compressed_size"); - uncompressedSize += rs.getLong("uncompressed_size"); - } - } - } - - Set shardIds = shardIdSet.build(); - if (shardIds.size() != shardUuids.size()) { - throw transactionConflict(); - } - - ShardDao dao = shardDaoSupplier.attach(handle); - dao.insertDeletedShards(shardUuids); - - String where = " WHERE shard_id IN (" + args + ")"; - String deleteFromShardNodes = "DELETE FROM shard_nodes " + where; - String deleteFromShards = "DELETE FROM shards " + where; - String deleteFromShardIndex = "DELETE FROM " + shardIndexTable(tableId) + where; - - try (PreparedStatement statement = handle.getConnection().prepareStatement(deleteFromShardNodes)) { - bindLongs(statement, shardIds); - statement.executeUpdate(); - } - - for (String sql : asList(deleteFromShards, deleteFromShardIndex)) { - try (PreparedStatement statement = handle.getConnection().prepareStatement(sql)) { - bindLongs(statement, shardIds); - if (statement.executeUpdate() != shardIds.size()) { - throw transactionConflict(); - } - } - } - - return new ShardStats(rowCount, compressedSize, uncompressedSize); - } - - private static void bindUuids(PreparedStatement statement, Iterable uuids) - throws SQLException - { - int i = 1; - for (UUID uuid : uuids) { - statement.setBytes(i, uuidToBytes(uuid)); - i++; - } - } - - private static void bindLongs(PreparedStatement statement, Iterable values) - throws SQLException - { - int i = 1; - for (long value : values) { - statement.setLong(i, value); - i++; - } - } - - private static void insertShardsAndIndex(long tableId, List columns, Collection shards, Map nodeIds, Handle handle) - throws SQLException - { - if (shards.isEmpty()) { - return; - } - boolean bucketed = shards.iterator().next().getBucketNumber().isPresent(); - - Connection connection = handle.getConnection(); - try (IndexInserter indexInserter = new IndexInserter(connection, tableId, columns)) { - for (List batch : partition(shards, batchSize(connection))) { - List shardIds = insertShards(connection, tableId, batch); - - if (!bucketed) { - insertShardNodes(connection, nodeIds, shardIds, batch); - } - - for (int i = 0; i < batch.size(); i++) { - ShardInfo shard = batch.get(i); - Set shardNodes = shard.getNodeIdentifiers().stream() - .map(nodeIds::get) - .collect(toSet()); - indexInserter.insert( - shardIds.get(i), - shard.getShardUuid(), - shard.getBucketNumber(), - shardNodes, - shard.getColumnStats()); - } - indexInserter.execute(); - } - } - } - - private static int batchSize(Connection connection) - { - // H2 does not return generated keys properly - // https://github.com/h2database/h2database/issues/156 - return (connection instanceof JdbcConnection) ? 1 : 1000; - } - - private Map toNodeIdMap(Collection shards) - { - Set identifiers = shards.stream() - .map(ShardInfo::getNodeIdentifiers) - .flatMap(Collection::stream) - .collect(toSet()); - return Maps.toMap(identifiers, this::getOrCreateNodeId); - } - - @Override - public ShardMetadata getShard(UUID shardUuid) - { - return dao.getShard(shardUuid); - } - - @Override - public Set getNodeShards(String nodeIdentifier) - { - return dao.getNodeShards(nodeIdentifier, null); - } - - @Override - public Set getNodeShards(String nodeIdentifier, long tableId) - { - return dao.getNodeShards(nodeIdentifier, tableId); - } - - @Override - public ResultIterator getShardNodes(long tableId, TupleDomain effectivePredicate) - { - return new ShardIterator(tableId, false, Optional.empty(), effectivePredicate, dbi); - } - - @Override - public ResultIterator getShardNodesBucketed(long tableId, boolean merged, List bucketToNode, TupleDomain effectivePredicate) - { - return new ShardIterator(tableId, merged, Optional.of(bucketToNode), effectivePredicate, dbi); - } - - @Override - public void replaceShardAssignment(long tableId, UUID shardUuid, String nodeIdentifier, boolean gracePeriod) - { - if (gracePeriod && (nanosSince(startTime).compareTo(startupGracePeriod) < 0)) { - throw new TrinoException(SERVER_STARTING_UP, "Cannot reassign shards while server is starting"); - } - - int nodeId = getOrCreateNodeId(nodeIdentifier); - - runTransaction(dbi, handle -> { - ShardDao dao = shardDaoSupplier.attach(handle); - - Set oldAssignments = new HashSet<>(fetchLockedNodeIds(handle, tableId, shardUuid)); - updateNodeIds(handle, tableId, shardUuid, ImmutableSet.of(nodeId)); - - dao.deleteShardNodes(shardUuid, oldAssignments); - dao.insertShardNode(shardUuid, nodeId); - return null; - }); - } - - @Override - public Map getNodeBytes() - { - return dao.getNodeSizes().stream() - .collect(toMap(NodeSize::getNodeIdentifier, NodeSize::getSizeInBytes)); - } - - @Override - public long beginTransaction() - { - return dao.insertTransaction(); - } - - @Override - public void rollbackTransaction(long transactionId) - { - dao.finalizeTransaction(transactionId, false); - } - - @Override - public void createBuckets(long distributionId, int bucketCount) - { - Iterator nodeIterator = cyclingShuffledIterator(getNodeIdentifiers()); - - List bucketNumbers = new ArrayList<>(); - List nodeIds = new ArrayList<>(); - for (int bucket = 0; bucket < bucketCount; bucket++) { - bucketNumbers.add(bucket); - nodeIds.add(getOrCreateNodeId(nodeIterator.next())); - } - - runIgnoringConstraintViolation(() -> dao.insertBuckets(distributionId, bucketNumbers, nodeIds)); - } - - @Override - public List getBucketAssignments(long distributionId) - { - try { - return bucketAssignmentsCache.getUnchecked(distributionId); - } - catch (UncheckedExecutionException e) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - throw e; - } - } - - @Override - public void updateBucketAssignment(long distributionId, int bucketNumber, String nodeId) - { - dao.updateBucketNode(distributionId, bucketNumber, getOrCreateNodeId(nodeId)); - } - - @Override - public List getDistributions() - { - return dao.listActiveDistributions(); - } - - @Override - public long getDistributionSizeInBytes(long distributionId) - { - return dao.getDistributionSizeBytes(distributionId); - } - - @Override - public List getBucketNodes(long distibutionId) - { - return dao.getBucketNodes(distibutionId); - } - - @Override - public Set getExistingShardUuids(long tableId, Set shardUuids) - { - try (Handle handle = dbi.open()) { - String args = Joiner.on(",").join(nCopies(shardUuids.size(), "?")); - String selectShards = format( - "SELECT shard_uuid FROM %s WHERE shard_uuid IN (%s)", - shardIndexTable(tableId), args); - - ImmutableSet.Builder existingShards = ImmutableSet.builder(); - try (PreparedStatement statement = handle.getConnection().prepareStatement(selectShards)) { - bindUuids(statement, shardUuids); - try (ResultSet rs = statement.executeQuery()) { - while (rs.next()) { - existingShards.add(uuidFromBytes(rs.getBytes("shard_uuid"))); - } - } - } - return existingShards.build(); - } - catch (SQLException e) { - throw new RuntimeException(e); - } - } - - private List getBuckets(long distributionId) - { - return dao.getBucketNodes(distributionId); - } - - private List loadBucketAssignments(long distributionId) - { - Set nodeIds = getNodeIdentifiers(); - List bucketNodes = getBuckets(distributionId); - BucketReassigner reassigner = new BucketReassigner(nodeIds, bucketNodes); - - List assignments = new ArrayList<>(nCopies(bucketNodes.size(), null)); - TrinoException limiterException = null; - Set offlineNodes = new HashSet<>(); - - for (BucketNode bucketNode : bucketNodes) { - int bucket = bucketNode.getBucketNumber(); - String nodeId = bucketNode.getNodeIdentifier(); - - if (!nodeIds.contains(nodeId)) { - if (nanosSince(startTime).compareTo(startupGracePeriod) < 0) { - throw new TrinoException(SERVER_STARTING_UP, "Cannot reassign buckets while server is starting"); - } - - try { - if (offlineNodes.add(nodeId)) { - assignmentLimiter.checkAssignFrom(nodeId); - } - } - catch (TrinoException e) { - if (limiterException == null) { - limiterException = e; - } - continue; - } - - String oldNodeId = nodeId; - nodeId = reassigner.getNextReassignmentDestination(); - dao.updateBucketNode(distributionId, bucket, getOrCreateNodeId(nodeId)); - log.info("Reassigned bucket %s for distribution ID %s from %s to %s", bucket, distributionId, oldNodeId, nodeId); - } - - verify(assignments.set(bucket, nodeId) == null, "Duplicate bucket"); - } - - if (limiterException != null) { - throw limiterException; - } - - return ImmutableList.copyOf(assignments); - } - - private Set getNodeIdentifiers() - { - Set nodeIds = nodeSupplier.getWorkerNodes().stream() - .map(Node::getNodeIdentifier) - .collect(toSet()); - if (nodeIds.isEmpty()) { - throw new TrinoException(NO_NODES_AVAILABLE, "No nodes available for bucket assignments"); - } - return nodeIds; - } - - private int getOrCreateNodeId(String nodeIdentifier) - { - try { - return nodeIdCache.getUnchecked(nodeIdentifier); - } - catch (UncheckedExecutionException e) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - throw e; - } - } - - private int loadNodeId(String nodeIdentifier) - { - Integer id = dao.getNodeId(nodeIdentifier); - if (id != null) { - return id; - } - - // creating a node is idempotent - runIgnoringConstraintViolation(() -> dao.insertNode(nodeIdentifier)); - - id = dao.getNodeId(nodeIdentifier); - if (id == null) { - throw new TrinoException(GENERIC_INTERNAL_ERROR, "node does not exist after insert"); - } - return id; - } - - private Duration nanosSince(long nanos) - { - return new Duration(ticker.read() - nanos, NANOSECONDS); - } - - private static List insertShards(Connection connection, long tableId, List shards) - throws SQLException - { - String sql = "" + - "INSERT INTO shards (shard_uuid, table_id, create_time, row_count, compressed_size, uncompressed_size, xxhash64, bucket_number)\n" + - "VALUES (?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?)"; - - try (PreparedStatement statement = connection.prepareStatement(sql, RETURN_GENERATED_KEYS)) { - for (ShardInfo shard : shards) { - statement.setBytes(1, uuidToBytes(shard.getShardUuid())); - statement.setLong(2, tableId); - statement.setLong(3, shard.getRowCount()); - statement.setLong(4, shard.getCompressedSize()); - statement.setLong(5, shard.getUncompressedSize()); - statement.setLong(6, shard.getXxhash64()); - bindOptionalInt(statement, 7, shard.getBucketNumber()); - statement.addBatch(); - } - statement.executeBatch(); - - ImmutableList.Builder builder = ImmutableList.builder(); - try (ResultSet keys = statement.getGeneratedKeys()) { - while (keys.next()) { - builder.add(keys.getLong(1)); - } - } - List shardIds = builder.build(); - - if (shardIds.size() != shards.size()) { - throw new TrinoException(RAPTOR_ERROR, "Wrong number of generated keys for inserted shards"); - } - return shardIds; - } - } - - private static void insertShardNodes(Connection connection, Map nodeIds, List shardIds, List shards) - throws SQLException - { - checkArgument(shardIds.size() == shards.size(), "lists are not the same size"); - String sql = "INSERT INTO shard_nodes (shard_id, node_id) VALUES (?, ?)"; - try (PreparedStatement statement = connection.prepareStatement(sql)) { - for (int i = 0; i < shards.size(); i++) { - for (String identifier : shards.get(i).getNodeIdentifiers()) { - statement.setLong(1, shardIds.get(i)); - statement.setInt(2, nodeIds.get(identifier)); - statement.addBatch(); - } - } - statement.executeBatch(); - } - } - - private static Collection fetchLockedNodeIds(Handle handle, long tableId, UUID shardUuid) - { - String sql = format( - "SELECT node_ids FROM %s WHERE shard_uuid = ? FOR UPDATE", - shardIndexTable(tableId)); - - byte[] nodeArray = handle.createQuery(sql) - .bind(0, uuidToBytes(shardUuid)) - .mapTo(byte[].class) - .one(); - - return intArrayFromBytes(nodeArray); - } - - private static void updateNodeIds(Handle handle, long tableId, UUID shardUuid, Set nodeIds) - { - String sql = format( - "UPDATE %s SET node_ids = ? WHERE shard_uuid = ?", - shardIndexTable(tableId)); - - handle.execute(sql, intArrayToBytes(nodeIds), uuidToBytes(shardUuid)); - } - - private static void lockTable(Handle handle, long tableId) - { - if (handle.attach(MetadataDao.class).getLockedTableId(tableId) == null) { - throw transactionConflict(); - } - } - - private static TrinoException transactionConflict() - { - return new TrinoException(TRANSACTION_CONFLICT, "Table was updated by a different transaction. Please retry the operation."); - } - - public static String shardIndexTable(long tableId) - { - return INDEX_TABLE_PREFIX + tableId; - } - - public static String minColumn(long columnId) - { - checkArgument(columnId >= 0, "invalid columnId %s", columnId); - return format("c%s_min", columnId); - } - - public static String maxColumn(long columnId) - { - checkArgument(columnId >= 0, "invalid columnId %s", columnId); - return format("c%s_max", columnId); - } - - private static String sqlColumnType(Type type) - { - JDBCType jdbcType = jdbcType(type); - if (jdbcType != null) { - switch (jdbcType) { - case BOOLEAN: - return "boolean"; - case BIGINT: - return "bigint"; - case DOUBLE: - return "double"; - case INTEGER: - return "int"; - case VARBINARY: - return format("varbinary(%s)", MAX_BINARY_INDEX_SIZE); - default: - break; - } - } - return null; - } - - private static Iterator cyclingShuffledIterator(Collection collection) - { - List list = new ArrayList<>(collection); - Collections.shuffle(list); - return Iterables.cycle(list).iterator(); - } - - private static ShardStats shardStats(Collection shards) - { - return new ShardStats( - shards.stream().mapToLong(ShardInfo::getRowCount).sum(), - shards.stream().mapToLong(ShardInfo::getCompressedSize).sum(), - shards.stream().mapToLong(ShardInfo::getUncompressedSize).sum()); - } - - private static class ShardStats - { - private final long rowCount; - private final long compressedSize; - private final long uncompressedSize; - - public ShardStats(long rowCount, long compressedSize, long uncompressedSize) - { - this.rowCount = rowCount; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - } - - public long getRowCount() - { - return rowCount; - } - - public long getCompressedSize() - { - return compressedSize; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardRecorder.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardRecorder.java deleted file mode 100644 index 801a570e6c82..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/DatabaseShardRecorder.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.TrinoException; - -import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; - -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.runIgnoringConstraintViolation; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class DatabaseShardRecorder - implements ShardRecorder -{ - private static final Logger log = Logger.get(DatabaseShardRecorder.class); - - private final ShardDao dao; - - @Inject - public DatabaseShardRecorder(DaoSupplier shardDaoSupplier) - { - this.dao = shardDaoSupplier.onDemand(); - } - - @Override - public void recordCreatedShard(long transactionId, UUID shardUuid) - { - int maxAttempts = 5; - for (int attempt = 1; attempt <= maxAttempts; attempt++) { - try { - runIgnoringConstraintViolation(() -> dao.insertCreatedShard(shardUuid, transactionId)); - return; - } - catch (TrinoException e) { - if (attempt == maxAttempts) { - throw e; - } - log.warn(e, "Failed to insert created shard on attempt %s, will retry", attempt); - try { - long millis = attempt * 2000L; - MILLISECONDS.sleep(millis + ThreadLocalRandom.current().nextLong(0, millis)); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw metadataError(ie); - } - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Distribution.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Distribution.java deleted file mode 100644 index 9ee87bb9f269..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Distribution.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.json.JsonCodec; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeManager; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; -import java.util.Optional; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.json.JsonCodec.listJsonCodec; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class Distribution -{ - private static final JsonCodec> LIST_CODEC = listJsonCodec(String.class); - - private final long id; - private final Optional name; - private final List columnTypes; - private final int bucketCount; - - public Distribution(long id, Optional name, List columnTypes, int bucketCount) - { - this.id = id; - this.name = requireNonNull(name, "name is null"); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.bucketCount = bucketCount; - } - - public long getId() - { - return id; - } - - public Optional getName() - { - return name; - } - - public List getColumnTypes() - { - return columnTypes; - } - - public int getBucketCount() - { - return bucketCount; - } - - public static class Mapper - implements RowMapper - { - private final TypeManager typeManager; - - @Inject - public Mapper(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public Distribution map(ResultSet rs, StatementContext ctx) - throws SQLException - { - List types = LIST_CODEC.fromJson(rs.getString("column_types")).stream() - .map(TypeId::of) - .map(typeManager::getType) - .collect(toImmutableList()); - - return new Distribution( - rs.getLong("distribution_id"), - Optional.ofNullable(rs.getString("distribution_name")), - types, - rs.getInt("bucket_count")); - } - } - - public static String serializeColumnTypes(List columnTypes) - { - return LIST_CODEC.toJson(columnTypes.stream() - .map(type -> type.getTypeId().getId()) - .collect(toList())); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ForMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ForMetadata.java deleted file mode 100644 index ee2b31918ddb..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ForMetadata.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.inject.BindingAnnotation; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -@Retention(RUNTIME) -@Target({FIELD, PARAMETER, METHOD}) -@BindingAnnotation -public @interface ForMetadata -{ -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2DatabaseConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2DatabaseConfig.java deleted file mode 100644 index e00ef7eea310..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2DatabaseConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import jakarta.validation.constraints.NotNull; - -public class H2DatabaseConfig -{ - private String filename; - - @NotNull - public String getFilename() - { - return filename; - } - - @Config("metadata.db.filename") - public H2DatabaseConfig setFilename(String filename) - { - this.filename = filename; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2ShardDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2ShardDao.java deleted file mode 100644 index 1dc04f70737d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/H2ShardDao.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.sqlobject.customizer.Bind; -import org.jdbi.v3.sqlobject.statement.SqlBatch; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.sql.Timestamp; -import java.util.UUID; - -public interface H2ShardDao - extends ShardDao -{ - @Override - @SqlBatch("MERGE INTO deleted_shards (shard_uuid, delete_time)\n" + - "VALUES (:shardUuid, CURRENT_TIMESTAMP)") - void insertDeletedShards(@Bind("shardUuid") Iterable shardUuids); - - @Override - @SqlUpdate("DELETE FROM transactions\n" + - "WHERE end_time < :maxEndTime\n" + - " AND successful IN (TRUE, FALSE)\n" + - " AND transaction_id NOT IN (SELECT transaction_id FROM created_shards)\n" + - "LIMIT " + CLEANUP_TRANSACTIONS_BATCH_SIZE) - int deleteOldCompletedTransactions(Timestamp maxEndTime); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/IndexInserter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/IndexInserter.java deleted file mode 100644 index aeadbdaade5a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/IndexInserter.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.jdbi.v3.core.Jdbi; - -import java.sql.Connection; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.OptionalInt; -import java.util.Set; -import java.util.StringJoiner; -import java.util.UUID; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isHiddenColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.metadata.ShardPredicate.bindValue; -import static io.trino.plugin.raptor.legacy.storage.ColumnIndexStatsUtils.jdbcType; -import static io.trino.plugin.raptor.legacy.util.ArrayUtil.intArrayToBytes; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidToBytes; - -class IndexInserter - implements AutoCloseable -{ - private final boolean bucketed; - private final List columns; - private final Map indexes; - private final Map types; - private final PreparedStatement statement; - - public IndexInserter(Connection connection, long tableId, List columns) - throws SQLException - { - this.bucketed = Jdbi.open(connection) - .select("SELECT distribution_id IS NOT NULL FROM tables WHERE table_id = ?", tableId) - .mapTo(boolean.class) - .one(); - - ImmutableList.Builder columnBuilder = ImmutableList.builder(); - ImmutableMap.Builder indexBuilder = ImmutableMap.builder(); - ImmutableMap.Builder typeBuilder = ImmutableMap.builder(); - StringJoiner nameJoiner = new StringJoiner(", "); - StringJoiner valueJoiner = new StringJoiner(", "); - int index = 1; - - nameJoiner.add("shard_id").add("shard_uuid"); - valueJoiner.add("?").add("?").add("?"); - index += 3; - - if (bucketed) { - nameJoiner.add("bucket_number"); - } - else { - nameJoiner.add("node_ids"); - } - - for (ColumnInfo column : columns) { - JDBCType jdbcType = jdbcType(column.getType()); - if (jdbcType == null) { - continue; - } - - long columnId = column.getColumnId(); - if (isHiddenColumn(columnId)) { - continue; - } - - columnBuilder.add(column); - nameJoiner.add(minColumn(columnId)); - nameJoiner.add(maxColumn(columnId)); - valueJoiner.add("?").add("?"); - - indexBuilder.put(columnId, index); - index += 2; - - typeBuilder.put(columnId, jdbcType); - } - - this.columns = columnBuilder.build(); - this.indexes = indexBuilder.buildOrThrow(); - this.types = typeBuilder.buildOrThrow(); - - String sql = "" + - "INSERT INTO " + shardIndexTable(tableId) + "\n" + - "(" + nameJoiner + ")\n" + - "VALUES (" + valueJoiner + ")"; - - this.statement = connection.prepareStatement(sql); - } - - @Override - public void close() - throws SQLException - { - statement.close(); - } - - public void insert(long shardId, UUID shardUuid, OptionalInt bucketNumber, Set nodeIds, List stats) - throws SQLException - { - statement.setLong(1, shardId); - statement.setBytes(2, uuidToBytes(shardUuid)); - - if (bucketed) { - checkArgument(bucketNumber.isPresent(), "shard bucket missing for bucketed table"); - statement.setInt(3, bucketNumber.getAsInt()); - } - else { - checkArgument(bucketNumber.isEmpty(), "shard bucket present for non-bucketed table"); - statement.setBytes(3, intArrayToBytes(nodeIds)); - } - - for (ColumnInfo column : columns) { - int index = indexes.get(column.getColumnId()); - int type = types.get(column.getColumnId()).getVendorTypeNumber(); - statement.setNull(index, type); - statement.setNull(index + 1, type); - } - - for (ColumnStats column : stats) { - if (!indexes.containsKey(column.getColumnId())) { - // the column no longer exists in the table - continue; - } - int index = indexes.get(column.getColumnId()); - JDBCType type = types.get(column.getColumnId()); - bindValue(statement, type, convert(column.getMin()), index); - bindValue(statement, type, convert(column.getMax()), index + 1); - } - - statement.addBatch(); - } - - public void execute() - throws SQLException - { - statement.executeBatch(); - } - - private static Object convert(Object value) - { - if (value instanceof String) { - return utf8Slice((String) value); - } - return value; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/JdbcDatabaseConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/JdbcDatabaseConfig.java deleted file mode 100644 index 0ff540f8631b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/JdbcDatabaseConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import jakarta.validation.constraints.NotNull; - -public class JdbcDatabaseConfig -{ - private String url; - - @NotNull - public String getUrl() - { - return url; - } - - @Config("metadata.db.url") - public JdbcDatabaseConfig setUrl(String url) - { - this.url = url; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataConfig.java deleted file mode 100644 index 55a1dca3e251..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataConfig.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; -import jakarta.validation.constraints.NotNull; - -import static java.util.concurrent.TimeUnit.MINUTES; - -public class MetadataConfig -{ - private Duration startupGracePeriod = new Duration(5, MINUTES); - private Duration reassignmentDelay = new Duration(0, MINUTES); - private Duration reassignmentInterval = new Duration(0, MINUTES); - - @NotNull - public Duration getStartupGracePeriod() - { - return startupGracePeriod; - } - - @Config("raptor.startup-grace-period") - @ConfigDescription("Minimum uptime before allowing bucket or shard reassignments") - public MetadataConfig setStartupGracePeriod(Duration startupGracePeriod) - { - this.startupGracePeriod = startupGracePeriod; - return this; - } - - @NotNull - public Duration getReassignmentDelay() - { - return reassignmentDelay; - } - - @Config("raptor.reassignment-delay") - @ConfigDescription("Minimum delay before allowing reassignments for a node") - public MetadataConfig setReassignmentDelay(Duration reassignmentDelay) - { - this.reassignmentDelay = reassignmentDelay; - return this; - } - - @NotNull - public Duration getReassignmentInterval() - { - return reassignmentInterval; - } - - @Config("raptor.reassignment-interval") - @ConfigDescription("Minimum interval between reassignments for different nodes") - public MetadataConfig setReassignmentInterval(Duration reassignmentInterval) - { - this.reassignmentInterval = reassignmentInterval; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataDao.java deleted file mode 100644 index 49c3cb7ad5c9..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MetadataDao.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.plugin.raptor.legacy.metadata.Table.TableMapper; -import io.trino.spi.connector.SchemaTableName; -import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; -import org.jdbi.v3.sqlobject.config.RegisterRowMapper; -import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.util.List; -import java.util.Set; - -@RegisterConstructorMapper(ColumnMetadataRow.class) -@RegisterConstructorMapper(TableMetadataRow.class) -@RegisterConstructorMapper(TableStatsRow.class) -@RegisterRowMapper(SchemaTableNameMapper.class) -@RegisterRowMapper(TableMapper.class) -@RegisterRowMapper(ViewResult.Mapper.class) -public interface MetadataDao -{ - String TABLE_INFORMATION_SELECT = "" + - "SELECT t.table_id, t.distribution_id, d.distribution_name, d.bucket_count, t.temporal_column_id, t.organization_enabled\n" + - "FROM tables t\n" + - "LEFT JOIN distributions d ON (t.distribution_id = d.distribution_id)\n"; - - String TABLE_COLUMN_SELECT = "" + - "SELECT t.schema_name, t.table_name,\n" + - " c.column_id, c.column_name, c.data_type, c.ordinal_position,\n" + - " c.bucket_ordinal_position, c.sort_ordinal_position,\n" + - " t.temporal_column_id = c.column_id AS temporal\n" + - "FROM tables t\n" + - "JOIN columns c ON (t.table_id = c.table_id)\n"; - - @SqlQuery(TABLE_INFORMATION_SELECT + - "WHERE t.table_id = :tableId") - Table getTableInformation(long tableId); - - @SqlQuery(TABLE_INFORMATION_SELECT + - "WHERE t.schema_name = :schemaName\n" + - " AND t.table_name = :tableName") - Table getTableInformation( - String schemaName, - String tableName); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE t.table_id = :tableId\n" + - " AND c.column_id = :columnId\n" + - "ORDER BY c.ordinal_position\n") - TableColumn getTableColumn( - long tableId, - long columnId); - - @SqlQuery("SELECT schema_name, table_name\n" + - "FROM tables\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)") - List listTables( - String schemaName); - - @SqlQuery("SELECT DISTINCT schema_name FROM tables") - List listSchemaNames(); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL)\n" + - "ORDER BY schema_name, table_name, ordinal_position") - List listTableColumns( - String schemaName, - String tableName); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE t.table_id = :tableId\n" + - "ORDER BY c.ordinal_position") - List listTableColumns(long tableId); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE t.table_id = :tableId\n" + - " AND c.sort_ordinal_position IS NOT NULL\n" + - "ORDER BY c.sort_ordinal_position") - List listSortColumns(long tableId); - - @SqlQuery(TABLE_COLUMN_SELECT + - "WHERE t.table_id = :tableId\n" + - " AND c.bucket_ordinal_position IS NOT NULL\n" + - "ORDER BY c.bucket_ordinal_position") - List listBucketColumns(long tableId); - - @SqlQuery("SELECT schema_name, table_name, data\n" + - "FROM views\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)") - List listViews( - String schemaName); - - @SqlQuery("SELECT schema_name, table_name, data\n" + - "FROM views\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL)\n" + - "ORDER BY schema_name, table_name\n") - List getViews( - String schemaName, - String tableName); - - @SqlUpdate("INSERT INTO tables (\n" + - " schema_name, table_name, compaction_enabled, organization_enabled, distribution_id,\n" + - " create_time, update_time, table_version,\n" + - " shard_count, row_count, compressed_size, uncompressed_size)\n" + - "VALUES (\n" + - " :schemaName, :tableName, :compactionEnabled, :organizationEnabled, :distributionId,\n" + - " :createTime, :createTime, 0,\n" + - " 0, 0, 0, 0)\n") - @GetGeneratedKeys - long insertTable( - String schemaName, - String tableName, - boolean compactionEnabled, - boolean organizationEnabled, - Long distributionId, - long createTime); - - @SqlUpdate("UPDATE tables SET\n" + - " update_time = :updateTime\n" + - ", table_version = table_version + 1\n" + - "WHERE table_id = :tableId") - void updateTableVersion( - long tableId, - long updateTime); - - @SqlUpdate("UPDATE tables SET\n" + - " shard_count = shard_count + :shardCount \n" + - ", row_count = row_count + :rowCount\n" + - ", compressed_size = compressed_size + :compressedSize\n" + - ", uncompressed_size = uncompressed_size + :uncompressedSize\n" + - "WHERE table_id = :tableId") - void updateTableStats( - long tableId, - long shardCount, - long rowCount, - long compressedSize, - long uncompressedSize); - - @SqlUpdate("INSERT INTO columns (table_id, column_id, column_name, ordinal_position, data_type, sort_ordinal_position, bucket_ordinal_position)\n" + - "VALUES (:tableId, :columnId, :columnName, :ordinalPosition, :dataType, :sortOrdinalPosition, :bucketOrdinalPosition)") - void insertColumn( - long tableId, - long columnId, - String columnName, - int ordinalPosition, - String dataType, - Integer sortOrdinalPosition, - Integer bucketOrdinalPosition); - - @SqlUpdate("UPDATE tables SET\n" + - " schema_name = :newSchemaName\n" + - ", table_name = :newTableName\n" + - "WHERE table_id = :tableId") - void renameTable( - long tableId, - String newSchemaName, - String newTableName); - - @SqlUpdate("UPDATE columns SET column_name = :target\n" + - "WHERE table_id = :tableId\n" + - " AND column_id = :columnId") - void renameColumn( - long tableId, - long columnId, - String target); - - @SqlUpdate("DELETE FROM columns\n" + - " WHERE table_id = :tableId\n" + - " AND column_id = :columnId") - void dropColumn( - long tableId, - long columnId); - - @SqlUpdate("INSERT INTO views (schema_name, table_name, data)\n" + - "VALUES (:schemaName, :tableName, :data)") - void insertView( - String schemaName, - String tableName, - String data); - - @SqlUpdate("DELETE FROM tables WHERE table_id = :tableId") - int dropTable(long tableId); - - @SqlUpdate("DELETE FROM columns WHERE table_id = :tableId") - int dropColumns(long tableId); - - @SqlUpdate("DELETE FROM views\n" + - "WHERE schema_name = :schemaName\n" + - " AND table_name = :tableName") - int dropView( - String schemaName, - String tableName); - - @SqlQuery("SELECT temporal_column_id\n" + - "FROM tables\n" + - "WHERE table_id = :tableId") - Long getTemporalColumnId(long tableId); - - @SqlUpdate("UPDATE tables SET\n" + - "temporal_column_id = :columnId\n" + - "WHERE table_id = :tableId") - void updateTemporalColumnId( - long tableId, - long columnId); - - @SqlQuery("SELECT compaction_enabled AND maintenance_blocked IS NULL\n" + - "FROM tables\n" + - "WHERE table_id = :tableId") - boolean isCompactionEligible(long tableId); - - @SqlQuery("SELECT table_id FROM tables WHERE table_id = :tableId FOR UPDATE") - Long getLockedTableId(long tableId); - - @SqlQuery("SELECT distribution_id, distribution_name, column_types, bucket_count\n" + - "FROM distributions\n" + - "WHERE distribution_id = :distributionId") - Distribution getDistribution(long distributionId); - - @SqlQuery("SELECT distribution_id, distribution_name, column_types, bucket_count\n" + - "FROM distributions\n" + - "WHERE distribution_name = :distributionName") - Distribution getDistribution(String distributionName); - - @SqlUpdate("INSERT INTO distributions (distribution_name, column_types, bucket_count)\n" + - "VALUES (:distributionName, :columnTypes, :bucketCount)") - @GetGeneratedKeys - long insertDistribution( - String distributionName, - String columnTypes, - int bucketCount); - - @SqlQuery("SELECT table_id, schema_name, table_name, temporal_column_id, distribution_name, bucket_count, organization_enabled\n" + - "FROM tables\n" + - "LEFT JOIN distributions\n" + - "ON tables.distribution_id = distributions.distribution_id\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL)\n" + - "ORDER BY table_id") - List getTableMetadataRows( - String schemaName, - String tableName); - - @SqlQuery("SELECT table_id, column_id, column_name, sort_ordinal_position, bucket_ordinal_position\n" + - "FROM columns\n" + - "WHERE table_id IN (\n" + - " SELECT table_id\n" + - " FROM tables\n" + - " WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL))\n" + - "ORDER BY table_id") - List getColumnMetadataRows( - String schemaName, - String tableName); - - @SqlQuery("SELECT schema_name, table_name, create_time, update_time, table_version,\n" + - " shard_count, row_count, compressed_size, uncompressed_size\n" + - "FROM tables\n" + - "WHERE (schema_name = :schemaName OR :schemaName IS NULL)\n" + - " AND (table_name = :tableName OR :tableName IS NULL)\n" + - "ORDER BY schema_name, table_name") - List getTableStatsRows( - String schemaName, - String tableName); - - @SqlQuery("SELECT table_id\n" + - "FROM tables\n" + - "WHERE organization_enabled\n" + - " AND maintenance_blocked IS NULL\n" + - " AND table_id IN\n" + - " (SELECT table_id\n" + - " FROM columns\n" + - " WHERE sort_ordinal_position IS NOT NULL)") - Set getOrganizationEligibleTables(); - - @SqlUpdate("UPDATE tables SET maintenance_blocked = CURRENT_TIMESTAMP\n" + - "WHERE table_id = :tableId\n" + - " AND maintenance_blocked IS NULL") - void blockMaintenance(long tableId); - - @SqlUpdate("UPDATE tables SET maintenance_blocked = NULL\n" + - "WHERE table_id = :tableId") - void unblockMaintenance(long tableId); - - @SqlQuery("SELECT maintenance_blocked IS NOT NULL\n" + - "FROM tables\n" + - "WHERE table_id = :tableId\n" + - "FOR UPDATE") - boolean isMaintenanceBlockedLocked(long tableId); - - @SqlUpdate("UPDATE tables SET maintenance_blocked = NULL\n" + - "WHERE maintenance_blocked IS NOT NULL") - void unblockAllMaintenance(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MySqlShardDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MySqlShardDao.java deleted file mode 100644 index 0df2cee91ea8..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/MySqlShardDao.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.sqlobject.customizer.Bind; -import org.jdbi.v3.sqlobject.statement.SqlBatch; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.sql.Timestamp; -import java.util.UUID; - -public interface MySqlShardDao - extends ShardDao -{ - @Override - @SqlUpdate("DELETE x\n" + - "FROM shard_nodes x\n" + - "JOIN shards USING (shard_id)\n" + - "WHERE table_id = :tableId") - void dropShardNodes(long tableId); - - @Override - @SqlBatch("INSERT IGNORE INTO deleted_shards (shard_uuid, delete_time)\n" + - "VALUES (:shardUuid, CURRENT_TIMESTAMP)") - void insertDeletedShards(@Bind("shardUuid") Iterable shardUuids); - - @Override - // 'order by' is needed in this statement in order to make it compatible with statement-based replication - @SqlUpdate("DELETE FROM transactions\n" + - "WHERE end_time < :maxEndTime\n" + - " AND successful IN (TRUE, FALSE)\n" + - " AND transaction_id NOT IN (SELECT transaction_id FROM created_shards)\n" + - "ORDER BY end_time, transaction_id\n" + - "LIMIT " + CLEANUP_TRANSACTIONS_BATCH_SIZE) - int deleteOldCompletedTransactions(Timestamp maxEndTime); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/NodeSize.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/NodeSize.java deleted file mode 100644 index 9f17495881bb..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/NodeSize.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.core.mapper.reflect.ColumnName; - -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class NodeSize -{ - private final String nodeIdentifier; - private final long sizeInBytes; - - public NodeSize(String nodeIdentifier, @ColumnName("bytes") long sizeInBytes) - { - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - checkArgument(sizeInBytes >= 0, "sizeInBytes must be >= 0"); - this.sizeInBytes = sizeInBytes; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } - - public long getSizeInBytes() - { - return sizeInBytes; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if ((o == null) || (getClass() != o.getClass())) { - return false; - } - NodeSize nodeSize = (NodeSize) o; - return (sizeInBytes == nodeSize.sizeInBytes) && - Objects.equals(nodeIdentifier, nodeSize.nodeIdentifier); - } - - @Override - public int hashCode() - { - return Objects.hash(nodeIdentifier, sizeInBytes); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("nodeIdentifier", nodeIdentifier) - .add("sizeInBytes", sizeInBytes) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/RaptorNode.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/RaptorNode.java deleted file mode 100644 index 162385a36c2f..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/RaptorNode.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import static java.util.Objects.requireNonNull; - -public class RaptorNode -{ - private final int nodeId; - private final String nodeIdentifier; - - public RaptorNode(int nodeId, String nodeIdentifier) - { - this.nodeId = nodeId; - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - } - - public int getNodeId() - { - return nodeId; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDao.java deleted file mode 100644 index 00a7b54d7b6e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDao.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -public interface SchemaDao -{ - @SqlUpdate("CREATE TABLE IF NOT EXISTS distributions (\n" + - " distribution_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + - " distribution_name VARCHAR(255),\n" + - " column_types TEXT NOT NULL,\n" + - " bucket_count INT NOT NULL,\n" + - " UNIQUE (distribution_name)\n" + - ")") - void createTableDistributions(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS tables (\n" + - " table_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + - " schema_name VARCHAR(255) NOT NULL,\n" + - " table_name VARCHAR(255) NOT NULL,\n" + - " temporal_column_id BIGINT,\n" + - " compaction_enabled BOOLEAN NOT NULL,\n" + - " organization_enabled BOOLEAN NOT NULL,\n" + - " distribution_id BIGINT,\n" + - " create_time BIGINT NOT NULL,\n" + - " update_time BIGINT NOT NULL,\n" + - " table_version BIGINT NOT NULL,\n" + - " shard_count BIGINT NOT NULL,\n" + - " row_count BIGINT NOT NULL,\n" + - " compressed_size BIGINT NOT NULL,\n" + - " uncompressed_size BIGINT NOT NULL,\n" + - " maintenance_blocked DATETIME,\n" + - " UNIQUE (schema_name, table_name),\n" + - " UNIQUE (distribution_id, table_id),\n" + - " UNIQUE (maintenance_blocked, table_id),\n" + - " FOREIGN KEY (distribution_id) REFERENCES distributions (distribution_id)\n" + - ")") - void createTableTables(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS columns (\n" + - " table_id BIGINT NOT NULL,\n" + - " column_id BIGINT NOT NULL,\n" + - " column_name VARCHAR(255) NOT NULL,\n" + - " ordinal_position INT NOT NULL,\n" + - " data_type VARCHAR(255) NOT NULL,\n" + - " sort_ordinal_position INT,\n" + - " bucket_ordinal_position INT,\n" + - " PRIMARY KEY (table_id, column_id),\n" + - " UNIQUE (table_id, column_name),\n" + - " UNIQUE (table_id, ordinal_position),\n" + - " UNIQUE (table_id, sort_ordinal_position),\n" + - " UNIQUE (table_id, bucket_ordinal_position),\n" + - " FOREIGN KEY (table_id) REFERENCES tables (table_id)\n" + - ")") - void createTableColumns(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS views (\n" + - " schema_name VARCHAR(255) NOT NULL,\n" + - " table_name VARCHAR(255) NOT NULL,\n" + - " data TEXT NOT NULL,\n" + - " PRIMARY KEY (schema_name, table_name)\n" + - ")") - void createTableViews(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS nodes (\n" + - " node_id INT PRIMARY KEY AUTO_INCREMENT,\n" + - " node_identifier VARCHAR(255) NOT NULL,\n" + - " UNIQUE (node_identifier)\n" + - ")") - void createTableNodes(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS shards (\n" + - " shard_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " table_id BIGINT NOT NULL,\n" + - " bucket_number INT,\n" + - " create_time DATETIME NOT NULL,\n" + - " row_count BIGINT NOT NULL,\n" + - " compressed_size BIGINT NOT NULL,\n" + - " uncompressed_size BIGINT NOT NULL,\n" + - " xxhash64 BIGINT NOT NULL,\n" + - " UNIQUE (shard_uuid),\n" + - // include a covering index organized by table_id - " UNIQUE (table_id, bucket_number, shard_id, shard_uuid, create_time, row_count, compressed_size, uncompressed_size, xxhash64),\n" + - " FOREIGN KEY (table_id) REFERENCES tables (table_id)\n" + - ")") - void createTableShards(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS shard_nodes (\n" + - " shard_id BIGINT NOT NULL,\n" + - " node_id INT NOT NULL,\n" + - " PRIMARY KEY (shard_id, node_id),\n" + - " UNIQUE (node_id, shard_id),\n" + - " FOREIGN KEY (shard_id) REFERENCES shards (shard_id),\n" + - " FOREIGN KEY (node_id) REFERENCES nodes (node_id)\n" + - ")") - void createTableShardNodes(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS external_batches (\n" + - " external_batch_id VARCHAR(255) PRIMARY KEY,\n" + - " successful BOOLEAN NOT NULL\n" + - ")") - void createTableExternalBatches(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS transactions (\n" + - " transaction_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + - " successful BOOLEAN,\n" + - " start_time DATETIME NOT NULL,\n" + - " end_time DATETIME,\n" + - " UNIQUE (successful, start_time, transaction_id),\n" + - " UNIQUE (end_time, transaction_id, successful)\n" + - ")") - void createTableTransactions(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS created_shards (\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " transaction_id BIGINT NOT NULL,\n" + - " PRIMARY KEY (shard_uuid),\n" + - " UNIQUE (transaction_id, shard_uuid),\n" + - " FOREIGN KEY (transaction_id) REFERENCES transactions (transaction_id)\n" + - ")") - void createTableCreatedShards(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS deleted_shards (\n" + - " shard_uuid BINARY(16) PRIMARY KEY,\n" + - " delete_time DATETIME NOT NULL,\n" + - " UNIQUE (delete_time, shard_uuid)\n" + - ")") - void createTableDeletedShards(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS buckets (\n" + - " distribution_id BIGINT NOT NULL,\n" + - " bucket_number INT NOT NULL,\n" + - " node_id INT NOT NULL,\n" + - " PRIMARY KEY (distribution_id, bucket_number),\n" + - " UNIQUE (node_id, distribution_id, bucket_number),\n" + - " FOREIGN KEY (distribution_id) REFERENCES distributions (distribution_id),\n" + - " FOREIGN KEY (node_id) REFERENCES nodes (node_id)\n" + - ")") - void createTableBuckets(); - - @SqlUpdate("CREATE TABLE IF NOT EXISTS shard_organizer_jobs (\n" + - " node_identifier VARCHAR(255) NOT NULL,\n" + - " table_id BIGINT NOT NULL,\n" + - " last_start_time BIGINT,\n" + - " PRIMARY KEY (node_identifier, table_id),\n" + - " UNIQUE (table_id, node_identifier),\n" + - " FOREIGN KEY (table_id) REFERENCES tables (table_id)\n" + - ")") - void createTableShardOrganizerJobs(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDaoUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDaoUtil.java deleted file mode 100644 index 5442248a1441..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaDaoUtil.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import org.jdbi.v3.core.ConnectionException; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; - -import java.util.concurrent.TimeUnit; - -public final class SchemaDaoUtil -{ - private static final Logger log = Logger.get(SchemaDaoUtil.class); - - private SchemaDaoUtil() {} - - public static void createTablesWithRetry(Jdbi dbi) - { - Duration delay = new Duration(2, TimeUnit.SECONDS); - while (true) { - try (Handle handle = dbi.open()) { - createTables(handle.attach(SchemaDao.class)); - return; - } - catch (ConnectionException e) { - log.warn("Failed to connect to database. Will retry again in %s. Exception: %s", delay, e.getMessage()); - sleep(delay); - } - } - } - - private static void createTables(SchemaDao dao) - { - dao.createTableDistributions(); - dao.createTableTables(); - dao.createTableColumns(); - dao.createTableViews(); - dao.createTableNodes(); - dao.createTableShards(); - dao.createTableShardNodes(); - dao.createTableExternalBatches(); - dao.createTableTransactions(); - dao.createTableCreatedShards(); - dao.createTableDeletedShards(); - dao.createTableBuckets(); - dao.createTableShardOrganizerJobs(); - } - - private static void sleep(Duration duration) - { - try { - Thread.sleep(duration.toMillis()); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaTableNameMapper.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaTableNameMapper.java deleted file mode 100644 index 61451b0e28b3..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/SchemaTableNameMapper.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.spi.connector.SchemaTableName; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; - -public class SchemaTableNameMapper - implements RowMapper -{ - @Override - public SchemaTableName map(ResultSet r, StatementContext ctx) - throws SQLException - { - return new SchemaTableName( - r.getString("schema_name"), - r.getString("table_name")); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleaner.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleaner.java deleted file mode 100644 index 6fa8169e00cc..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleaner.java +++ /dev/null @@ -1,519 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Ticker; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.storage.StorageService; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.NodeManager; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.io.File; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.collect.Sets.difference; -import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.metadata.ShardDao.CLEANABLE_SHARDS_BATCH_SIZE; -import static io.trino.plugin.raptor.legacy.metadata.ShardDao.CLEANUP_TRANSACTIONS_BATCH_SIZE; -import static java.lang.Math.min; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.runAsync; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toSet; - -public class ShardCleaner -{ - private static final Logger log = Logger.get(ShardCleaner.class); - - private final ShardDao dao; - private final String currentNode; - private final boolean coordinator; - private final Ticker ticker; - private final StorageService storageService; - private final Optional backupStore; - private final Duration maxTransactionAge; - private final Duration transactionCleanerInterval; - private final Duration localCleanerInterval; - private final Duration localCleanTime; - private final Duration backupCleanerInterval; - private final Duration backupCleanTime; - private final ScheduledExecutorService scheduler; - private final ExecutorService backupExecutor; - private final Duration maxCompletedTransactionAge; - - private final AtomicBoolean started = new AtomicBoolean(); - - private final CounterStat transactionJobErrors = new CounterStat(); - private final CounterStat backupJobErrors = new CounterStat(); - private final CounterStat localJobErrors = new CounterStat(); - private final CounterStat backupShardsQueued = new CounterStat(); - private final CounterStat backupShardsCleaned = new CounterStat(); - private final CounterStat localShardsCleaned = new CounterStat(); - - @GuardedBy("this") - private final Map shardsToClean = new HashMap<>(); - - @Inject - public ShardCleaner( - DaoSupplier shardDaoSupplier, - Ticker ticker, - NodeManager nodeManager, - StorageService storageService, - Optional backupStore, - ShardCleanerConfig config) - { - this( - shardDaoSupplier, - nodeManager.getCurrentNode().getNodeIdentifier(), - nodeManager.getCurrentNode().isCoordinator(), - ticker, - storageService, - backupStore, - config.getMaxTransactionAge(), - config.getTransactionCleanerInterval(), - config.getLocalCleanerInterval(), - config.getLocalCleanTime(), - config.getBackupCleanerInterval(), - config.getBackupCleanTime(), - config.getBackupDeletionThreads(), - config.getMaxCompletedTransactionAge()); - } - - public ShardCleaner( - DaoSupplier shardDaoSupplier, - String currentNode, - boolean coordinator, - Ticker ticker, - StorageService storageService, - Optional backupStore, - Duration maxTransactionAge, - Duration transactionCleanerInterval, - Duration localCleanerInterval, - Duration localCleanTime, - Duration backupCleanerInterval, - Duration backupCleanTime, - int backupDeletionThreads, - Duration maxCompletedTransactionAge) - { - this.dao = shardDaoSupplier.onDemand(); - this.currentNode = requireNonNull(currentNode, "currentNode is null"); - this.coordinator = coordinator; - this.ticker = requireNonNull(ticker, "ticker is null"); - this.storageService = requireNonNull(storageService, "storageService is null"); - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.maxTransactionAge = requireNonNull(maxTransactionAge, "maxTransactionAge"); - this.transactionCleanerInterval = requireNonNull(transactionCleanerInterval, "transactionCleanerInterval is null"); - this.localCleanerInterval = requireNonNull(localCleanerInterval, "localCleanerInterval is null"); - this.localCleanTime = requireNonNull(localCleanTime, "localCleanTime is null"); - this.backupCleanerInterval = requireNonNull(backupCleanerInterval, "backupCleanerInterval is null"); - this.backupCleanTime = requireNonNull(backupCleanTime, "backupCleanTime is null"); - this.scheduler = newScheduledThreadPool(2, daemonThreadsNamed("shard-cleaner-%s")); - this.backupExecutor = newFixedThreadPool(backupDeletionThreads, daemonThreadsNamed("shard-cleaner-backup-%s")); - this.maxCompletedTransactionAge = requireNonNull(maxCompletedTransactionAge, "maxCompletedTransactionAge is null"); - } - - @PostConstruct - public void start() - { - if (!started.getAndSet(true)) { - startJobs(); - } - } - - @PreDestroy - public void shutdown() - { - scheduler.shutdownNow(); - backupExecutor.shutdownNow(); - } - - @Managed - @Nested - public CounterStat getTransactionJobErrors() - { - return transactionJobErrors; - } - - @Managed - @Nested - public CounterStat getBackupJobErrors() - { - return backupJobErrors; - } - - @Managed - @Nested - public CounterStat getLocalJobErrors() - { - return localJobErrors; - } - - @Managed - @Nested - public CounterStat getBackupShardsQueued() - { - return backupShardsQueued; - } - - @Managed - @Nested - public CounterStat getBackupShardsCleaned() - { - return backupShardsCleaned; - } - - @Managed - @Nested - public CounterStat getLocalShardsCleaned() - { - return localShardsCleaned; - } - - private void startJobs() - { - if (coordinator) { - startTransactionCleanup(); - if (backupStore.isPresent()) { - scheduleBackupCleanup(); - } - } - - // We can only delete local shards if the backup store is present, - // since there is a race condition between shards getting created - // on a worker and being committed (referenced) in the database. - if (backupStore.isPresent()) { - scheduleLocalCleanup(); - } - } - - private void startTransactionCleanup() - { - scheduler.scheduleWithFixedDelay(() -> { - try { - abortOldTransactions(); - deleteOldCompletedTransactions(); - deleteOldShards(); - } - catch (Throwable t) { - log.error(t, "Error cleaning transactions"); - transactionJobErrors.update(1); - } - }, 0, transactionCleanerInterval.toMillis(), MILLISECONDS); - } - - private void scheduleBackupCleanup() - { - scheduler.scheduleWithFixedDelay(this::runBackupCleanup, 0, backupCleanerInterval.toMillis(), MILLISECONDS); - } - - private void scheduleLocalCleanup() - { - Set local = getLocalShards(); - scheduler.submit(() -> { - waitJitterTime(); - runLocalCleanupImmediately(local); - }); - - scheduler.scheduleWithFixedDelay(() -> { - waitJitterTime(); - runLocalCleanup(); - }, 0, localCleanerInterval.toMillis(), MILLISECONDS); - } - - private synchronized void runBackupCleanup() - { - try { - cleanBackupShards(); - } - catch (Throwable t) { - log.error(t, "Error cleaning backup shards"); - backupJobErrors.update(1); - } - } - - private void waitJitterTime() - { - try { - // jitter to avoid overloading database - long interval = this.localCleanerInterval.roundTo(SECONDS); - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval)); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private synchronized void runLocalCleanupImmediately(Set local) - { - try { - cleanLocalShardsImmediately(local); - } - catch (Throwable t) { - log.error(t, "Error cleaning local shards"); - localJobErrors.update(1); - } - } - - private synchronized void runLocalCleanup() - { - try { - cleanLocalShards(); - } - catch (Throwable t) { - log.error(t, "Error cleaning local shards"); - localJobErrors.update(1); - } - } - - @Managed - public void startBackupCleanup() - { - scheduler.submit(this::runBackupCleanup); - } - - @Managed - public void startLocalCleanup() - { - scheduler.submit(this::runLocalCleanup); - } - - @Managed - public void startLocalCleanupImmediately() - { - scheduler.submit(() -> { - runLocalCleanupImmediately(getLocalShards()); - }); - } - - @VisibleForTesting - void abortOldTransactions() - { - dao.abortOldTransactions(maxTimestamp(maxTransactionAge)); - } - - @VisibleForTesting - void deleteOldShards() - { - while (!Thread.currentThread().isInterrupted()) { - List shards = dao.getOldCreatedShardsBatch(); - if (shards.isEmpty()) { - break; - } - dao.insertDeletedShards(shards); - dao.deleteCreatedShards(shards); - backupShardsQueued.update(shards.size()); - } - } - - @VisibleForTesting - void deleteOldCompletedTransactions() - { - while (!Thread.currentThread().isInterrupted()) { - int deleted = dao.deleteOldCompletedTransactions(maxTimestamp(maxCompletedTransactionAge)); - if (deleted < CLEANUP_TRANSACTIONS_BATCH_SIZE) { - break; - } - } - } - - @VisibleForTesting - synchronized Set getLocalShards() - { - return storageService.getStorageShards(); - } - - @VisibleForTesting - synchronized void cleanLocalShardsImmediately(Set local) - { - // get shards assigned to the local node - Set assigned = dao.getNodeShards(currentNode, null).stream() - .map(ShardMetadata::getShardUuid) - .collect(toSet()); - - // only delete local files that are not assigned - Set deletions = Sets.difference(local, assigned); - - for (UUID uuid : deletions) { - deleteFile(storageService.getStorageFile(uuid)); - } - - localShardsCleaned.update(deletions.size()); - log.info("Cleaned %s local shards immediately", deletions.size()); - } - - @VisibleForTesting - synchronized void cleanLocalShards() - { - // find all files on the local node - Set local = getLocalShards(); - - // get shards assigned to the local node - Set assigned = dao.getNodeShards(currentNode, null).stream() - .map(ShardMetadata::getShardUuid) - .collect(toSet()); - - // un-mark previously marked files that are now assigned - for (UUID uuid : assigned) { - shardsToClean.remove(uuid); - } - - // mark all files that are not assigned - for (UUID uuid : local) { - if (!assigned.contains(uuid)) { - shardsToClean.putIfAbsent(uuid, ticker.read()); - } - } - - // delete files marked earlier than the clean interval - long threshold = ticker.read() - localCleanTime.roundTo(NANOSECONDS); - Set deletions = shardsToClean.entrySet().stream() - .filter(entry -> entry.getValue() < threshold) - .map(Map.Entry::getKey) - .collect(toSet()); - if (deletions.isEmpty()) { - return; - } - - for (UUID uuid : deletions) { - deleteFile(storageService.getStorageFile(uuid)); - shardsToClean.remove(uuid); - } - - localShardsCleaned.update(deletions.size()); - log.info("Cleaned %s local shards", deletions.size()); - } - - @VisibleForTesting - void cleanBackupShards() - { - Set processing = newConcurrentHashSet(); - BlockingQueue completed = new LinkedBlockingQueue<>(); - boolean fill = true; - - while (!Thread.currentThread().isInterrupted()) { - // get a new batch if any completed and we are under the batch size - Set uuids = ImmutableSet.of(); - if (fill && (processing.size() < CLEANABLE_SHARDS_BATCH_SIZE)) { - uuids = dao.getCleanableShardsBatch(maxTimestamp(backupCleanTime)); - fill = false; - } - if (uuids.isEmpty() && processing.isEmpty()) { - break; - } - - // skip any that are already processing and mark remaining as processing - uuids = ImmutableSet.copyOf(difference(uuids, processing)); - processing.addAll(uuids); - - // execute deletes - for (UUID uuid : uuids) { - runAsync(() -> backupStore.get().deleteShard(uuid), backupExecutor) - .thenAccept(v -> completed.add(uuid)) - .whenComplete((v, e) -> { - if (e != null) { - log.error(e, "Error cleaning backup shard: %s", uuid); - backupJobErrors.update(1); - processing.remove(uuid); - } - }); - } - - // get the next batch of completed deletes - int desired = min(100, processing.size()); - Collection done = drain(completed, desired, 100, MILLISECONDS); - if (done.isEmpty()) { - continue; - } - - // remove completed deletes from database - processing.removeAll(done); - dao.deleteCleanedShards(done); - backupShardsCleaned.update(done.size()); - fill = true; - } - } - - private static Collection drain(BlockingQueue queue, int desired, long timeout, TimeUnit unit) - { - long start = System.nanoTime(); - Collection result = new ArrayList<>(); - - while (true) { - queue.drainTo(result); - if (result.size() >= desired) { - return result; - } - - long elapsedNanos = System.nanoTime() - start; - long remainingNanos = unit.toNanos(timeout) - elapsedNanos; - if (remainingNanos <= 0) { - return result; - } - - try { - T value = queue.poll(remainingNanos, NANOSECONDS); - if (value != null) { - result.add(value); - } - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return result; - } - } - } - - private static Timestamp maxTimestamp(Duration duration) - { - return new Timestamp(System.currentTimeMillis() - duration.toMillis()); - } - - @SuppressWarnings("ResultOfMethodCallIgnored") - private static void deleteFile(File file) - { - file.delete(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleanerConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleanerConfig.java deleted file mode 100644 index 9c112b2773df..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardCleanerConfig.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; -import io.airlift.units.MaxDuration; -import io.airlift.units.MinDuration; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; - -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MINUTES; - -public class ShardCleanerConfig -{ - private Duration maxTransactionAge = new Duration(1, DAYS); - private Duration transactionCleanerInterval = new Duration(10, MINUTES); - private Duration localCleanerInterval = new Duration(1, HOURS); - private Duration localCleanTime = new Duration(4, HOURS); - private Duration backupCleanerInterval = new Duration(5, MINUTES); - private Duration backupCleanTime = new Duration(1, DAYS); - private int backupDeletionThreads = 50; - private Duration maxCompletedTransactionAge = new Duration(1, DAYS); - - @NotNull - @MinDuration("1m") - @MaxDuration("30d") - public Duration getMaxTransactionAge() - { - return maxTransactionAge; - } - - @Config("raptor.max-transaction-age") - @ConfigDescription("Maximum time a transaction may run before it is aborted") - public ShardCleanerConfig setMaxTransactionAge(Duration maxTransactionAge) - { - this.maxTransactionAge = maxTransactionAge; - return this; - } - - @NotNull - @MinDuration("1m") - public Duration getTransactionCleanerInterval() - { - return transactionCleanerInterval; - } - - @Config("raptor.transaction-cleaner-interval") - @ConfigDescription("How often to cleanup expired transactions") - public ShardCleanerConfig setTransactionCleanerInterval(Duration transactionCleanerInterval) - { - this.transactionCleanerInterval = transactionCleanerInterval; - return this; - } - - @NotNull - @MinDuration("1m") - public Duration getLocalCleanerInterval() - { - return localCleanerInterval; - } - - @Config("raptor.local-cleaner-interval") - @ConfigDescription("How often to discover local shards that need to be cleaned up") - public ShardCleanerConfig setLocalCleanerInterval(Duration localCleanerInterval) - { - this.localCleanerInterval = localCleanerInterval; - return this; - } - - @NotNull - public Duration getLocalCleanTime() - { - return localCleanTime; - } - - @Config("raptor.local-clean-time") - @ConfigDescription("How long to wait after discovery before cleaning local shards") - public ShardCleanerConfig setLocalCleanTime(Duration localCleanTime) - { - this.localCleanTime = localCleanTime; - return this; - } - - @NotNull - @MinDuration("1m") - public Duration getBackupCleanerInterval() - { - return backupCleanerInterval; - } - - @Config("raptor.backup-cleaner-interval") - @ConfigDescription("How often to check for backup shards that need to be cleaned up") - public ShardCleanerConfig setBackupCleanerInterval(Duration backupCleanerInterval) - { - this.backupCleanerInterval = backupCleanerInterval; - return this; - } - - @NotNull - public Duration getBackupCleanTime() - { - return backupCleanTime; - } - - @Config("raptor.backup-clean-time") - @ConfigDescription("How long to wait after deletion before cleaning backup shards") - public ShardCleanerConfig setBackupCleanTime(Duration backupCleanTime) - { - this.backupCleanTime = backupCleanTime; - return this; - } - - @Min(1) - public int getBackupDeletionThreads() - { - return backupDeletionThreads; - } - - @Config("raptor.backup-deletion-threads") - @ConfigDescription("Maximum number of threads to use for deleting shards from backup store") - public ShardCleanerConfig setBackupDeletionThreads(int backupDeletionThreads) - { - this.backupDeletionThreads = backupDeletionThreads; - return this; - } - - @NotNull - @MinDuration("1m") - @MaxDuration("30d") - public Duration getMaxCompletedTransactionAge() - { - return maxCompletedTransactionAge; - } - - @Config("raptor.max-completed-transaction-age") - @ConfigDescription("Maximum time a record of a successful or failed transaction is kept") - public ShardCleanerConfig setMaxCompletedTransactionAge(Duration maxCompletedTransactionAge) - { - this.maxCompletedTransactionAge = maxCompletedTransactionAge; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDao.java deleted file mode 100644 index f5ac3c33d94b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDao.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.plugin.raptor.legacy.util.UuidUtil.UuidArgumentFactory; -import io.trino.plugin.raptor.legacy.util.UuidUtil.UuidColumnMapper; -import org.jdbi.v3.sqlobject.config.RegisterArgumentFactory; -import org.jdbi.v3.sqlobject.config.RegisterColumnMapper; -import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; -import org.jdbi.v3.sqlobject.config.RegisterRowMapper; -import org.jdbi.v3.sqlobject.customizer.Bind; -import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; -import org.jdbi.v3.sqlobject.statement.SqlBatch; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.sql.Timestamp; -import java.util.List; -import java.util.Set; -import java.util.UUID; - -@RegisterArgumentFactory(UuidArgumentFactory.class) -@RegisterColumnMapper(UuidColumnMapper.class) -@RegisterConstructorMapper(BucketNode.class) -@RegisterConstructorMapper(NodeSize.class) -@RegisterConstructorMapper(RaptorNode.class) -@RegisterRowMapper(ShardMetadata.Mapper.class) -public interface ShardDao -{ - int CLEANABLE_SHARDS_BATCH_SIZE = 1000; - int CLEANUP_TRANSACTIONS_BATCH_SIZE = 10_000; - - String SHARD_METADATA_COLUMNS = "table_id, shard_id, shard_uuid, bucket_number, row_count, compressed_size, uncompressed_size, xxhash64"; - - @SqlUpdate("INSERT INTO nodes (node_identifier) VALUES (:nodeIdentifier)") - @GetGeneratedKeys - int insertNode(String nodeIdentifier); - - @SqlUpdate("INSERT INTO shard_nodes (shard_id, node_id)\n" + - "VALUES ((SELECT shard_id FROM shards WHERE shard_uuid = :shardUuid), :nodeId)") - void insertShardNode(UUID shardUuid, int nodeId); - - @SqlBatch("DELETE FROM shard_nodes\n" + - "WHERE shard_id = (SELECT shard_id FROM shards WHERE shard_uuid = :shardUuid)\n" + - " AND node_id = :nodeId") - void deleteShardNodes(UUID shardUuid, @Bind("nodeId") Iterable nodeIds); - - @SqlQuery("SELECT node_id FROM nodes WHERE node_identifier = :nodeIdentifier") - Integer getNodeId(String nodeIdentifier); - - @SqlQuery("SELECT node_identifier FROM nodes WHERE node_id = :nodeId") - String getNodeIdentifier(int nodeId); - - @SqlQuery("SELECT node_id, node_identifier FROM nodes") - List getNodes(); - - @SqlQuery("SELECT " + SHARD_METADATA_COLUMNS + " FROM shards WHERE shard_uuid = :shardUuid") - ShardMetadata getShard(UUID shardUuid); - - @SqlQuery("SELECT " + SHARD_METADATA_COLUMNS + "\n" + - "FROM (\n" + - " SELECT s.*\n" + - " FROM shards s\n" + - " JOIN shard_nodes sn ON (s.shard_id = sn.shard_id)\n" + - " JOIN nodes n ON (sn.node_id = n.node_id)\n" + - " WHERE n.node_identifier = :nodeIdentifier\n" + - " AND s.bucket_number IS NULL\n" + - " AND (s.table_id = :tableId OR :tableId IS NULL)\n" + - " UNION ALL\n" + - " SELECT s.*\n" + - " FROM shards s\n" + - " JOIN tables t ON (s.table_id = t.table_id)\n" + - " JOIN distributions d ON (t.distribution_id = d.distribution_id)\n" + - " JOIN buckets b ON (\n" + - " d.distribution_id = b.distribution_id AND\n" + - " s.bucket_number = b.bucket_number)\n" + - " JOIN nodes n ON (b.node_id = n.node_id)\n" + - " WHERE n.node_identifier = :nodeIdentifier\n" + - " AND (s.table_id = :tableId OR :tableId IS NULL)\n" + - ") x") - Set getNodeShards(String nodeIdentifier, Long tableId); - - @SqlQuery("SELECT n.node_identifier, x.bytes\n" + - "FROM (\n" + - " SELECT node_id, sum(compressed_size) bytes\n" + - " FROM (\n" + - " SELECT sn.node_id, s.compressed_size\n" + - " FROM shards s\n" + - " JOIN shard_nodes sn ON (s.shard_id = sn.shard_id)\n" + - " WHERE s.bucket_number IS NULL\n" + - " UNION ALL\n" + - " SELECT b.node_id, s.compressed_size\n" + - " FROM shards s\n" + - " JOIN tables t ON (s.table_id = t.table_id)\n" + - " JOIN distributions d ON (t.distribution_id = d.distribution_id)\n" + - " JOIN buckets b ON (\n" + - " d.distribution_id = b.distribution_id AND\n" + - " s.bucket_number = b.bucket_number)\n" + - " ) x\n" + - " GROUP BY node_id\n" + - ") x\n" + - "JOIN nodes n ON (x.node_id = n.node_id)") - Set getNodeSizes(); - - @SqlUpdate("DELETE FROM shard_nodes WHERE shard_id IN (\n" + - " SELECT shard_id\n" + - " FROM shards\n" + - " WHERE table_id = :tableId)") - void dropShardNodes(long tableId); - - @SqlUpdate("DELETE FROM shards WHERE table_id = :tableId") - void dropShards(long tableId); - - @SqlUpdate("INSERT INTO external_batches (external_batch_id, successful)\n" + - "VALUES (:externalBatchId, TRUE)") - void insertExternalBatch(String externalBatchId); - - @SqlQuery("SELECT count(*)\n" + - "FROM external_batches\n" + - "WHERE external_batch_id = :externalBatchId") - boolean externalBatchExists(String externalBatchId); - - @SqlUpdate("INSERT INTO transactions (start_time) VALUES (CURRENT_TIMESTAMP)") - @GetGeneratedKeys - long insertTransaction(); - - @SqlUpdate("UPDATE transactions SET\n" + - " successful = :successful\n" + - ", end_time = CURRENT_TIMESTAMP\n" + - "WHERE transaction_id = :transactionId\n" + - " AND successful IS NULL") - int finalizeTransaction( - long transactionId, - boolean successful); - - @SqlQuery("SELECT successful FROM transactions WHERE transaction_id = :transactionId") - Boolean transactionSuccessful(long transactionId); - - @SqlUpdate("UPDATE transactions SET\n" + - " successful = FALSE\n" + - ", end_time = CURRENT_TIMESTAMP\n" + - "WHERE successful IS NULL\n" + - " AND start_time < :maxStartTime") - void abortOldTransactions(Timestamp maxStartTime); - - @SqlUpdate("INSERT INTO created_shards (shard_uuid, transaction_id)\n" + - "VALUES (:shardUuid, :transactionId)") - void insertCreatedShard( - UUID shardUuid, - long transactionId); - - @SqlUpdate("DELETE FROM created_shards WHERE transaction_id = :transactionId") - void deleteCreatedShards(long transactionId); - - @SqlBatch("DELETE FROM created_shards WHERE shard_uuid = :shardUuid") - void deleteCreatedShards(@Bind("shardUuid") Iterable shardUuids); - - default void insertDeletedShards(Iterable shardUuids) - { - throw new UnsupportedOperationException(); - } - - @SqlUpdate("INSERT INTO deleted_shards (shard_uuid, delete_time)\n" + - "SELECT shard_uuid, CURRENT_TIMESTAMP\n" + - "FROM shards\n" + - "WHERE table_id = :tableId") - void insertDeletedShards(long tableId); - - @SqlQuery("SELECT s.shard_uuid\n" + - "FROM created_shards s\n" + - "JOIN transactions t ON (s.transaction_id = t.transaction_id)\n" + - "WHERE NOT t.successful\n" + - "LIMIT 10000") - List getOldCreatedShardsBatch(); - - @SqlQuery("SELECT shard_uuid\n" + - "FROM deleted_shards\n" + - "WHERE delete_time < :maxDeleteTime\n" + - "LIMIT " + CLEANABLE_SHARDS_BATCH_SIZE) - Set getCleanableShardsBatch(Timestamp maxDeleteTime); - - @SqlBatch("DELETE FROM deleted_shards WHERE shard_uuid = :shardUuid") - void deleteCleanedShards(@Bind("shardUuid") Iterable shardUuids); - - @SqlBatch("INSERT INTO buckets (distribution_id, bucket_number, node_id)\n" + - "VALUES (:distributionId, :bucketNumber, :nodeId)\n") - void insertBuckets( - long distributionId, - @Bind("bucketNumber") List bucketNumbers, - @Bind("nodeId") List nodeIds); - - @SqlQuery("SELECT b.bucket_number, n.node_identifier\n" + - "FROM buckets b\n" + - "JOIN nodes n ON (b.node_id = n.node_id)\n" + - "WHERE b.distribution_id = :distributionId\n" + - "ORDER BY b.bucket_number") - List getBucketNodes(long distributionId); - - @SqlQuery("SELECT distribution_id, distribution_name, column_types, bucket_count\n" + - "FROM distributions\n" + - "WHERE distribution_id IN (SELECT distribution_id FROM tables)") - List listActiveDistributions(); - - @SqlQuery("SELECT SUM(compressed_size)\n" + - "FROM tables\n" + - "WHERE distribution_id = :distributionId") - long getDistributionSizeBytes(long distributionId); - - @SqlUpdate("UPDATE buckets SET node_id = :nodeId\n" + - "WHERE distribution_id = :distributionId\n" + - " AND bucket_number = :bucketNumber") - void updateBucketNode( - long distributionId, - int bucketNumber, - int nodeId); - - default int deleteOldCompletedTransactions(Timestamp maxEndTime) - { - throw new UnsupportedOperationException(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDelta.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDelta.java deleted file mode 100644 index e53a37f58bc1..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardDelta.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardDelta -{ - private final List oldShardUuids; - private final List newShards; - - @JsonCreator - public ShardDelta( - @JsonProperty("oldShardUuids") List oldShardUuids, - @JsonProperty("newShards") List newShards) - { - this.oldShardUuids = ImmutableList.copyOf(requireNonNull(oldShardUuids, "oldShardUuids is null")); - this.newShards = ImmutableList.copyOf(requireNonNull(newShards, "newShards is null")); - } - - @JsonProperty - public List getOldShardUuids() - { - return oldShardUuids; - } - - @JsonProperty - public List getNewShards() - { - return newShards; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("oldShardUuids", oldShardUuids) - .add("newShards", newShards) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardInfo.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardInfo.java deleted file mode 100644 index 5050c376b69b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardInfo.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; - -import java.util.List; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public class ShardInfo -{ - private final UUID shardUuid; - private final OptionalInt bucketNumber; - private final Set nodeIdentifiers; - private final List columnStats; - private final long rowCount; - private final long compressedSize; - private final long uncompressedSize; - private final long xxhash64; - - @JsonCreator - public ShardInfo( - @JsonProperty("shardUuid") UUID shardUuid, - @JsonProperty("bucketNumber") OptionalInt bucketNumber, - @JsonProperty("nodeIdentifiers") Set nodeIdentifiers, - @JsonProperty("columnStats") List columnStats, - @JsonProperty("rowCount") long rowCount, - @JsonProperty("compressedSize") long compressedSize, - @JsonProperty("uncompressedSize") long uncompressedSize, - @JsonProperty("xxhash64") long xxhash64) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.nodeIdentifiers = ImmutableSet.copyOf(requireNonNull(nodeIdentifiers, "nodeIdentifiers is null")); - this.columnStats = ImmutableList.copyOf(requireNonNull(columnStats, "columnStats is null")); - - checkArgument(rowCount >= 0, "rowCount must be positive"); - checkArgument(compressedSize >= 0, "compressedSize must be positive"); - checkArgument(uncompressedSize >= 0, "uncompressedSize must be positive"); - this.rowCount = rowCount; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - - this.xxhash64 = xxhash64; - } - - @JsonProperty - public UUID getShardUuid() - { - return shardUuid; - } - - @JsonProperty - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - @JsonProperty - public Set getNodeIdentifiers() - { - return nodeIdentifiers; - } - - @JsonProperty - public List getColumnStats() - { - return columnStats; - } - - @JsonProperty - public long getRowCount() - { - return rowCount; - } - - @JsonProperty - public long getCompressedSize() - { - return compressedSize; - } - - @JsonProperty - public long getUncompressedSize() - { - return uncompressedSize; - } - - @JsonProperty - public long getXxhash64() - { - return xxhash64; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuid", shardUuid) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .add("nodeIdentifiers", nodeIdentifiers) - .add("columnStats", columnStats) - .add("rowCount", rowCount) - .add("compressedSize", compressedSize) - .add("uncompressedSize", uncompressedSize) - .add("xxhash64", format("%016x", xxhash64)) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardIterator.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardIterator.java deleted file mode 100644 index 22fcab27f5df..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardIterator.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.AbstractIterator; -import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.TrinoException; -import io.trino.spi.predicate.TupleDomain; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.result.ResultIterator; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; -import java.util.function.Function; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.util.ArrayUtil.intArrayFromBytes; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.enableStreamingResults; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static java.lang.String.format; -import static java.util.stream.Collectors.toSet; - -final class ShardIterator - extends AbstractIterator - implements ResultIterator -{ - private static final Logger log = Logger.get(ShardIterator.class); - private final Map nodeMap = new HashMap<>(); - - private final boolean merged; - private final List bucketToNode; - private final ShardDao dao; - private final Connection connection; - private final PreparedStatement statement; - private final ResultSet resultSet; - private boolean first = true; - - public ShardIterator( - long tableId, - boolean merged, - Optional> bucketToNode, - TupleDomain effectivePredicate, - Jdbi dbi) - { - this.merged = merged; - this.bucketToNode = bucketToNode.orElse(null); - ShardPredicate predicate = ShardPredicate.create(effectivePredicate); - - String sql; - if (bucketToNode.isPresent()) { - sql = "SELECT shard_uuid, bucket_number FROM %s WHERE %s ORDER BY bucket_number"; - } - else { - sql = "SELECT shard_uuid, node_ids FROM %s WHERE %s"; - } - sql = format(sql, shardIndexTable(tableId), predicate.getPredicate()); - - dao = onDemandDao(dbi, ShardDao.class); - fetchNodes(); - - try { - connection = dbi.open().getConnection(); - statement = connection.prepareStatement(sql); - enableStreamingResults(statement); - predicate.bind(statement); - log.debug("Running query: %s", statement); - resultSet = statement.executeQuery(); - } - catch (SQLException e) { - close(); - throw metadataError(e); - } - catch (Throwable t) { - close(); - throw t; - } - } - - @Override - protected BucketShards computeNext() - { - try { - return merged ? computeMerged() : compute(); - } - catch (SQLException e) { - throw metadataError(e); - } - } - - @SuppressWarnings({"UnusedDeclaration", "EmptyTryBlock"}) - @Override - public void close() - { - // use try-with-resources to close everything properly - try (Connection connection = this.connection; - Statement statement = this.statement; - ResultSet resultSet = this.resultSet) { - // do nothing - } - catch (SQLException _) { - } - } - - @Override - public StatementContext getContext() - { - throw new UnsupportedOperationException(); - } - - /** - * Compute split-per-shard (separate split for each shard). - */ - private BucketShards compute() - throws SQLException - { - if (!resultSet.next()) { - return endOfData(); - } - - UUID shardUuid = uuidFromBytes(resultSet.getBytes("shard_uuid")); - Set nodeIdentifiers; - OptionalInt bucketNumber = OptionalInt.empty(); - - if (bucketToNode != null) { - int bucket = resultSet.getInt("bucket_number"); - bucketNumber = OptionalInt.of(bucket); - nodeIdentifiers = ImmutableSet.of(getBucketNode(bucket)); - } - else { - List nodeIds = intArrayFromBytes(resultSet.getBytes("node_ids")); - nodeIdentifiers = getNodeIdentifiers(nodeIds, shardUuid); - } - - ShardNodes shard = new ShardNodes(shardUuid, nodeIdentifiers); - return new BucketShards(bucketNumber, ImmutableSet.of(shard)); - } - - /** - * Compute split-per-bucket (single split for all shards in a bucket). - */ - private BucketShards computeMerged() - throws SQLException - { - if (resultSet.isAfterLast()) { - return endOfData(); - } - if (first) { - first = false; - if (!resultSet.next()) { - return endOfData(); - } - } - - int bucketNumber = resultSet.getInt("bucket_number"); - ImmutableSet.Builder shards = ImmutableSet.builder(); - - do { - UUID shardUuid = uuidFromBytes(resultSet.getBytes("shard_uuid")); - int bucket = resultSet.getInt("bucket_number"); - Set nodeIdentifiers = ImmutableSet.of(getBucketNode(bucket)); - - shards.add(new ShardNodes(shardUuid, nodeIdentifiers)); - } - while (resultSet.next() && resultSet.getInt("bucket_number") == bucketNumber); - - return new BucketShards(OptionalInt.of(bucketNumber), shards.build()); - } - - private String getBucketNode(int bucket) - { - String node = bucketToNode.get(bucket); - if (node == null) { - throw new TrinoException(RAPTOR_ERROR, "No node mapping for bucket: " + bucket); - } - return node; - } - - private Set getNodeIdentifiers(List nodeIds, UUID shardUuid) - { - Function fetchNode = id -> fetchNode(id, shardUuid); - return nodeIds.stream() - .map(id -> nodeMap.computeIfAbsent(id, fetchNode)) - .collect(toSet()); - } - - private String fetchNode(int id, UUID shardUuid) - { - String node = dao.getNodeIdentifier(id); - if (node == null) { - throw new TrinoException(RAPTOR_ERROR, format("Missing node ID [%s] for shard: %s", id, shardUuid)); - } - return node; - } - - private void fetchNodes() - { - for (RaptorNode node : dao.getNodes()) { - nodeMap.put(node.getNodeId(), node.getNodeIdentifier()); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardManager.java deleted file mode 100644 index 13f18ee7dcd0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardManager.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.predicate.TupleDomain; -import org.jdbi.v3.core.result.ResultIterator; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -public interface ShardManager -{ - /** - * Create a table. - */ - void createTable(long tableId, List columns, boolean bucketed, OptionalLong temporalColumnId); - - /** - * Drop a table. - */ - void dropTable(long tableId); - - /** - * Add a column to the end of the table. - */ - void addColumn(long tableId, ColumnInfo column); - - /** - * Commit data for a table. - */ - void commitShards(long transactionId, long tableId, List columns, Collection shards, Optional externalBatchId, long updateTime); - - /** - * Replace oldShardsUuids with newShards. - */ - void replaceShardUuids(long transactionId, long tableId, List columns, Set oldShardUuids, Collection newShards, OptionalLong updateTime); - - /** - * Get shard metadata for a shard. - */ - ShardMetadata getShard(UUID shardUuid); - - /** - * Get shard metadata for shards on a given node. - */ - Set getNodeShards(String nodeIdentifier); - - /** - * Get shard metadata for shards on a given node. - */ - Set getNodeShards(String nodeIdentifier, long tableId); - - /** - * Return the shard nodes for a non-bucketed table. - */ - ResultIterator getShardNodes(long tableId, TupleDomain effectivePredicate); - - /** - * Return the shard nodes for a bucketed table. - */ - ResultIterator getShardNodesBucketed(long tableId, boolean merged, List bucketToNode, TupleDomain effectivePredicate); - - /** - * Remove all old shard assignments and assign a shard to a node - */ - void replaceShardAssignment(long tableId, UUID shardUuid, String nodeIdentifier, boolean gracePeriod); - - /** - * Get the number of bytes used by assigned shards per node. - */ - Map getNodeBytes(); - - /** - * Begin a transaction for creating shards. - * - * @return transaction ID - */ - long beginTransaction(); - - /** - * Rollback a transaction. - */ - void rollbackTransaction(long transactionId); - - /** - * Create initial bucket assignments for a distribution. - */ - void createBuckets(long distributionId, int bucketCount); - - /** - * Get map of buckets to node identifiers for a distribution. - */ - List getBucketAssignments(long distributionId); - - /** - * Change the node a bucket is assigned to. - */ - void updateBucketAssignment(long distributionId, int bucketNumber, String nodeId); - - /** - * Get all active distributions. - */ - List getDistributions(); - - /** - * Get total physical size of all tables in a distribution. - */ - long getDistributionSizeInBytes(long distributionId); - - /** - * Get list of bucket nodes for a distribution. - */ - List getBucketNodes(long distributionId); - - /** - * Return the subset of shard uuids that exist - */ - Set getExistingShardUuids(long tableId, Set shardUuids); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardMetadata.java deleted file mode 100644 index fd0d4c35bac6..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardMetadata.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.airlift.units.DataSize; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Objects; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.ToStringHelper; -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalInt; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalLong; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public class ShardMetadata -{ - private final long tableId; - private final long shardId; - private final UUID shardUuid; - private final OptionalInt bucketNumber; - private final long rowCount; - private final long compressedSize; - private final long uncompressedSize; - private final OptionalLong xxhash64; - private final OptionalLong rangeStart; - private final OptionalLong rangeEnd; - - public ShardMetadata( - long tableId, - long shardId, - UUID shardUuid, - OptionalInt bucketNumber, - long rowCount, - long compressedSize, - long uncompressedSize, - OptionalLong xxhash64, - OptionalLong rangeStart, - OptionalLong rangeEnd) - { - checkArgument(tableId > 0, "tableId must be > 0"); - checkArgument(shardId > 0, "shardId must be > 0"); - checkArgument(rowCount >= 0, "rowCount must be >= 0"); - checkArgument(compressedSize >= 0, "compressedSize must be >= 0"); - checkArgument(uncompressedSize >= 0, "uncompressedSize must be >= 0"); - - this.tableId = tableId; - this.shardId = shardId; - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.rowCount = rowCount; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - this.xxhash64 = requireNonNull(xxhash64, "xxhash64 is null"); - this.rangeStart = requireNonNull(rangeStart, "rangeStart is null"); - this.rangeEnd = requireNonNull(rangeEnd, "rangeEnd is null"); - } - - public long getTableId() - { - return tableId; - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public long getShardId() - { - return shardId; - } - - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - public long getRowCount() - { - return rowCount; - } - - public long getCompressedSize() - { - return compressedSize; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - - public OptionalLong getXxhash64() - { - return xxhash64; - } - - public OptionalLong getRangeStart() - { - return rangeStart; - } - - public OptionalLong getRangeEnd() - { - return rangeEnd; - } - - public ShardMetadata withTimeRange(long rangeStart, long rangeEnd) - { - return new ShardMetadata( - tableId, - shardId, - shardUuid, - bucketNumber, - rowCount, - compressedSize, - uncompressedSize, - xxhash64, - OptionalLong.of(rangeStart), - OptionalLong.of(rangeEnd)); - } - - @Override - public String toString() - { - ToStringHelper stringHelper = toStringHelper(this) - .add("tableId", tableId) - .add("shardId", shardId) - .add("shardUuid", shardUuid) - .add("rowCount", rowCount) - .add("compressedSize", DataSize.succinctBytes(compressedSize)) - .add("uncompressedSize", DataSize.succinctBytes(uncompressedSize)); - - if (bucketNumber.isPresent()) { - stringHelper.add("bucketNumber", bucketNumber.getAsInt()); - } - if (xxhash64.isPresent()) { - stringHelper.add("xxhash64", format("%16x", xxhash64.getAsLong())); - } - if (rangeStart.isPresent()) { - stringHelper.add("rangeStart", rangeStart.getAsLong()); - } - if (rangeEnd.isPresent()) { - stringHelper.add("rangeEnd", rangeEnd.getAsLong()); - } - return stringHelper.toString(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ShardMetadata that = (ShardMetadata) o; - return tableId == that.tableId && - shardId == that.shardId && - Objects.equals(bucketNumber, that.bucketNumber) && - rowCount == that.rowCount && - compressedSize == that.compressedSize && - uncompressedSize == that.uncompressedSize && - Objects.equals(xxhash64, that.xxhash64) && - Objects.equals(shardUuid, that.shardUuid) && - Objects.equals(rangeStart, that.rangeStart) && - Objects.equals(rangeEnd, that.rangeEnd); - } - - @Override - public int hashCode() - { - return Objects.hash( - tableId, - shardId, - shardUuid, - bucketNumber, - rowCount, - compressedSize, - uncompressedSize, - xxhash64, - rangeStart, - rangeEnd); - } - - public static class Mapper - implements RowMapper - { - @Override - public ShardMetadata map(ResultSet r, StatementContext ctx) - throws SQLException - { - return new ShardMetadata( - r.getLong("table_id"), - r.getLong("shard_id"), - uuidFromBytes(r.getBytes("shard_uuid")), - getOptionalInt(r, "bucket_number"), - r.getLong("row_count"), - r.getLong("compressed_size"), - r.getLong("uncompressed_size"), - getOptionalLong(r, "xxhash64"), - OptionalLong.empty(), - OptionalLong.empty()); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardNodes.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardNodes.java deleted file mode 100644 index abfc2f8e2732..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardNodes.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableSet; - -import java.util.Objects; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardNodes -{ - private final UUID shardUuid; - private final Set nodeIdentifiers; - - public ShardNodes(UUID shardUuid, Set nodeIdentifiers) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.nodeIdentifiers = ImmutableSet.copyOf(requireNonNull(nodeIdentifiers, "nodeIdentifiers is null")); - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public Set getNodeIdentifiers() - { - return nodeIdentifiers; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - ShardNodes other = (ShardNodes) obj; - return Objects.equals(this.shardUuid, other.shardUuid) && - Objects.equals(this.nodeIdentifiers, other.nodeIdentifiers); - } - - @Override - public int hashCode() - { - return Objects.hash(shardUuid, nodeIdentifiers); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuid", shardUuid) - .add("nodeIdentifiers", nodeIdentifiers) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardPredicate.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardPredicate.java deleted file mode 100644 index 881b585fab11..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardPredicate.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.TrinoException; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.Ranges; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; - -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.List; -import java.util.Map.Entry; -import java.util.StringJoiner; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.storage.ColumnIndexStatsUtils.jdbcType; -import static io.trino.plugin.raptor.legacy.storage.ShardStats.truncateIndexValue; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidStringToBytes; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -class ShardPredicate -{ - private final String predicate; - private final List types; - private final List values; - private static final int MAX_RANGE_COUNT = 100; - - private ShardPredicate(String predicate, List types, List values) - { - this.predicate = requireNonNull(predicate, "predicate is null"); - this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); - this.values = ImmutableList.copyOf(requireNonNull(values, "values is null")); - checkArgument(types.size() == values.size(), "types and values sizes do not match"); - } - - public String getPredicate() - { - return predicate; - } - - public void bind(PreparedStatement statement) - throws SQLException - { - for (int i = 0; i < types.size(); i++) { - JDBCType type = types.get(i); - Object value = values.get(i); - bindValue(statement, type, value, i + 1); - } - } - - @Override - public String toString() - { - return toStringHelper(this) - .addValue(predicate) - .toString(); - } - - public static ShardPredicate create(TupleDomain tupleDomain) - { - StringJoiner predicate = new StringJoiner(" AND ").setEmptyValue("true"); - ImmutableList.Builder types = ImmutableList.builder(); - ImmutableList.Builder values = ImmutableList.builder(); - - for (Entry entry : tupleDomain.getDomains().get().entrySet()) { - Domain domain = entry.getValue(); - if (domain.isNullAllowed() || domain.isAll()) { - continue; - } - RaptorColumnHandle handle = entry.getKey(); - Type type = handle.getColumnType(); - - JDBCType jdbcType = jdbcType(type); - if (jdbcType == null) { - continue; - } - - if (handle.isShardUuid()) { - predicate.add(createShardPredicate(types, values, domain, jdbcType)); - continue; - } - - if (!domain.getType().isOrderable()) { - continue; - } - - StringJoiner columnPredicate = new StringJoiner(" OR ", "(", ")").setEmptyValue("true"); - Ranges ranges = domain.getValues().getRanges(); - - // prevent generating complicated metadata queries - if (ranges.getRangeCount() > MAX_RANGE_COUNT) { - continue; - } - - for (Range range : ranges.getOrderedRanges()) { - String min; - String max; - if (handle.isBucketNumber()) { - min = "bucket_number"; - max = "bucket_number"; - } - else { - min = minColumn(handle.getColumnId()); - max = maxColumn(handle.getColumnId()); - } - - StringJoiner rangePredicate = new StringJoiner(" AND ", "(", ")").setEmptyValue("true"); - if (!range.isLowUnbounded()) { - rangePredicate.add(format("(%s >= ? OR %s IS NULL)", max, max)); - types.add(jdbcType); - values.add(range.getLowBoundedValue()); - } - if (!range.isHighUnbounded()) { - rangePredicate.add(format("(%s <= ? OR %s IS NULL)", min, min)); - types.add(jdbcType); - values.add(range.getHighBoundedValue()); - } - columnPredicate.add(rangePredicate.toString()); - } - predicate.add(columnPredicate.toString()); - } - return new ShardPredicate(predicate.toString(), types.build(), values.build()); - } - - private static String createShardPredicate(ImmutableList.Builder types, ImmutableList.Builder values, Domain domain, JDBCType jdbcType) - { - List ranges = domain.getValues().getRanges().getOrderedRanges(); - - // only apply predicates if all ranges are single values - if (ranges.isEmpty() || !ranges.stream().allMatch(Range::isSingleValue)) { - return "true"; - } - - ImmutableList.Builder valuesBuilder = ImmutableList.builder(); - ImmutableList.Builder typesBuilder = ImmutableList.builder(); - - StringJoiner rangePredicate = new StringJoiner(" OR "); - for (Range range : ranges) { - Slice uuidText = (Slice) range.getSingleValue(); - try { - Slice uuidBytes = uuidStringToBytes(uuidText); - typesBuilder.add(jdbcType); - valuesBuilder.add(uuidBytes); - } - catch (IllegalArgumentException e) { - return "true"; - } - rangePredicate.add("shard_uuid = ?"); - } - - types.addAll(typesBuilder.build()); - values.addAll(valuesBuilder.build()); - return rangePredicate.toString(); - } - - @VisibleForTesting - protected List getTypes() - { - return types; - } - - @VisibleForTesting - protected List getValues() - { - return values; - } - - public static void bindValue(PreparedStatement statement, JDBCType type, Object value, int index) - throws SQLException - { - if (value == null) { - statement.setNull(index, type.getVendorTypeNumber()); - return; - } - - switch (type) { - case BOOLEAN: - statement.setBoolean(index, (boolean) value); - return; - case INTEGER: - statement.setInt(index, ((Number) value).intValue()); - return; - case BIGINT: - statement.setLong(index, ((Number) value).longValue()); - return; - case DOUBLE: - statement.setDouble(index, ((Number) value).doubleValue()); - return; - case VARBINARY: - statement.setBytes(index, truncateIndexValue((Slice) value).getBytes()); - return; - default: - throw new TrinoException(GENERIC_INTERNAL_ERROR, "Unhandled type: " + type); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardRecorder.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardRecorder.java deleted file mode 100644 index e7c0661edd32..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ShardRecorder.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import java.util.UUID; - -public interface ShardRecorder -{ - void recordCreatedShard(long transactionId, UUID shardUuid); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Table.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Table.java deleted file mode 100644 index 90ee80608f02..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/Table.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getBoxedLong; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalInt; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalLong; -import static java.util.Objects.requireNonNull; - -public final class Table -{ - private final long tableId; - private final Optional distributionId; - private final Optional distributionName; - private final OptionalInt bucketCount; - private final OptionalLong temporalColumnId; - private final boolean organized; - - public Table( - long tableId, - Optional distributionId, - Optional distributionName, - OptionalInt bucketCount, - OptionalLong temporalColumnId, - boolean organized) - { - this.tableId = tableId; - this.distributionId = requireNonNull(distributionId, "distributionId is null"); - this.distributionName = requireNonNull(distributionName, "distributionName is null"); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.temporalColumnId = requireNonNull(temporalColumnId, "temporalColumnId is null"); - this.organized = organized; - } - - public long getTableId() - { - return tableId; - } - - public Optional getDistributionId() - { - return distributionId; - } - - public Optional getDistributionName() - { - return distributionName; - } - - public OptionalInt getBucketCount() - { - return bucketCount; - } - - public OptionalLong getTemporalColumnId() - { - return temporalColumnId; - } - - public boolean isOrganized() - { - return organized; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("tableId", tableId) - .add("distributionId", distributionId.orElse(null)) - .add("bucketCount", bucketCount.isPresent() ? bucketCount.getAsInt() : null) - .add("temporalColumnId", temporalColumnId.isPresent() ? temporalColumnId.getAsLong() : null) - .add("organized", organized) - .omitNullValues() - .toString(); - } - - public static class TableMapper - implements RowMapper - { - @Override - public Table map(ResultSet r, StatementContext ctx) - throws SQLException - { - return new Table( - r.getLong("table_id"), - Optional.ofNullable(getBoxedLong(r, "distribution_id")), - Optional.ofNullable(r.getString("distribution_name")), - getOptionalInt(r, "bucket_count"), - getOptionalLong(r, "temporal_column_id"), - r.getBoolean("organization_enabled")); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableColumn.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableColumn.java deleted file mode 100644 index af07d86df157..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableColumn.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.inject.Inject; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeManager; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.OptionalInt; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.getOptionalInt; -import static java.util.Objects.requireNonNull; - -public class TableColumn -{ - private final SchemaTableName table; - private final String columnName; - private final Type dataType; - private final long columnId; - private final int ordinalPosition; - private final OptionalInt bucketOrdinal; - private final OptionalInt sortOrdinal; - private final boolean temporal; - - public TableColumn(SchemaTableName table, String columnName, Type dataType, long columnId, int ordinalPosition, OptionalInt bucketOrdinal, OptionalInt sortOrdinal, boolean temporal) - { - this.table = requireNonNull(table, "table is null"); - this.columnName = requireNonNull(columnName, "columnName is null"); - this.dataType = requireNonNull(dataType, "dataType is null"); - this.columnId = columnId; - this.ordinalPosition = ordinalPosition; - this.bucketOrdinal = requireNonNull(bucketOrdinal, "bucketOrdinal is null"); - this.sortOrdinal = requireNonNull(sortOrdinal, "sortOrdinal is null"); - this.temporal = temporal; - } - - public SchemaTableName getTable() - { - return table; - } - - public String getColumnName() - { - return columnName; - } - - public Type getDataType() - { - return dataType; - } - - public long getColumnId() - { - return columnId; - } - - public int getOrdinalPosition() - { - return ordinalPosition; - } - - public OptionalInt getBucketOrdinal() - { - return bucketOrdinal; - } - - public OptionalInt getSortOrdinal() - { - return sortOrdinal; - } - - public boolean isTemporal() - { - return temporal; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("table", table) - .add("columnId", columnId) - .add("columnName", columnName) - .add("dataType", dataType) - .toString(); - } - - public ColumnMetadata toColumnMetadata() - { - return new ColumnMetadata(columnName, dataType); - } - - public ColumnInfo toColumnInfo() - { - return new ColumnInfo(columnId, dataType); - } - - public static class Mapper - implements RowMapper - { - private final TypeManager typeManager; - - @Inject - public Mapper(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public TableColumn map(ResultSet r, StatementContext ctx) - throws SQLException - { - SchemaTableName table = new SchemaTableName( - r.getString("schema_name"), - r.getString("table_name")); - - String typeName = r.getString("data_type"); - Type type = typeManager.getType(TypeId.of(typeName)); - - return new TableColumn( - table, - r.getString("column_name"), - type, - r.getLong("column_id"), - r.getInt("ordinal_position"), - getOptionalInt(r, "bucket_ordinal_position"), - getOptionalInt(r, "sort_ordinal_position"), - r.getBoolean("temporal")); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadata.java deleted file mode 100644 index a99242b1f728..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadata.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public final class TableMetadata -{ - private final long tableId; - private final List columns; - private final List sortColumnIds; - - public TableMetadata(long tableId, List columns, List sortColumnIds) - { - this.tableId = tableId; - this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); - this.sortColumnIds = ImmutableList.copyOf(requireNonNull(sortColumnIds, "sortColumnIds is null")); - } - - public long getTableId() - { - return tableId; - } - - public List getColumns() - { - return columns; - } - - public List getSortColumnIds() - { - return sortColumnIds; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TableMetadata that = (TableMetadata) o; - return tableId == that.tableId && - Objects.equals(columns, that.columns) && - Objects.equals(sortColumnIds, that.sortColumnIds); - } - - @Override - public int hashCode() - { - return Objects.hash(tableId, columns, sortColumnIds); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("tableId", tableId) - .add("columns", columns) - .add("sortColumnIds", sortColumnIds) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadataRow.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadataRow.java deleted file mode 100644 index 29469caeaf86..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableMetadataRow.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.core.mapper.reflect.ColumnName; - -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static java.util.Objects.requireNonNull; - -public class TableMetadataRow -{ - private final long tableId; - private final String schemaName; - private final String tableName; - private final OptionalLong temporalColumnId; - private final Optional distributionName; - private final OptionalInt bucketCount; - private final boolean organized; - - public TableMetadataRow( - long tableId, - String schemaName, - String tableName, - OptionalLong temporalColumnId, - Optional distributionName, - OptionalInt bucketCount, - @ColumnName("organization_enabled") boolean organized) - { - this.tableId = tableId; - this.schemaName = requireNonNull(schemaName, "schemaName is null"); - this.tableName = requireNonNull(tableName, "tableName is null"); - this.temporalColumnId = requireNonNull(temporalColumnId, "temporalColumnId is null"); - this.distributionName = requireNonNull(distributionName, "distributionName is null"); - this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); - this.organized = organized; - } - - public long getTableId() - { - return tableId; - } - - public String getSchemaName() - { - return schemaName; - } - - public String getTableName() - { - return tableName; - } - - public OptionalLong getTemporalColumnId() - { - return temporalColumnId; - } - - public Optional getDistributionName() - { - return distributionName; - } - - public OptionalInt getBucketCount() - { - return bucketCount; - } - - public boolean isOrganized() - { - return organized; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableStatsRow.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableStatsRow.java deleted file mode 100644 index 1d295dc185a4..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/TableStatsRow.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import static java.util.Objects.requireNonNull; - -public class TableStatsRow -{ - private final String schemaName; - private final String tableName; - private final long createTime; - private final long updateTime; - private final long tableVersion; - private final long shardCount; - private final long rowCount; - private final long compressedSize; - private final long uncompressedSize; - - public TableStatsRow( - String schemaName, - String tableName, - long createTime, - long updateTime, - long tableVersion, - long shardCount, - long rowCount, - long compressedSize, - long uncompressedSize) - { - this.schemaName = requireNonNull(schemaName, "schemaName is null"); - this.tableName = requireNonNull(tableName, "tableName is null"); - this.createTime = createTime; - this.updateTime = updateTime; - this.tableVersion = tableVersion; - this.shardCount = shardCount; - this.rowCount = rowCount; - this.compressedSize = compressedSize; - this.uncompressedSize = uncompressedSize; - } - - public String getSchemaName() - { - return schemaName; - } - - public String getTableName() - { - return tableName; - } - - public long getCreateTime() - { - return createTime; - } - - public long getUpdateTime() - { - return updateTime; - } - - public long getTableVersion() - { - return tableVersion; - } - - public long getShardCount() - { - return shardCount; - } - - public long getRowCount() - { - return rowCount; - } - - public long getCompressedSize() - { - return compressedSize; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ViewResult.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ViewResult.java deleted file mode 100644 index 76579db13967..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/metadata/ViewResult.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import io.trino.spi.connector.SchemaTableName; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import static java.util.Objects.requireNonNull; - -public class ViewResult -{ - private final SchemaTableName name; - private final String data; - - public ViewResult(SchemaTableName name, String data) - { - this.name = requireNonNull(name, "name is null"); - this.data = requireNonNull(data, "data is null"); - } - - public SchemaTableName getName() - { - return name; - } - - public String getData() - { - return data; - } - - public static class Mapper - implements RowMapper - { - @Override - public ViewResult map(ResultSet r, StatementContext ctx) - throws SQLException - { - SchemaTableName name = new SchemaTableName( - r.getString("schema_name"), - r.getString("table_name")); - return new ViewResult(name, r.getString("data")); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurity.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurity.java deleted file mode 100644 index bad1aacc2aaf..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurity.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -public enum RaptorSecurity -{ - ALLOW_ALL, - FILE, - READ_ONLY, -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityConfig.java deleted file mode 100644 index f62552835047..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import io.airlift.configuration.Config; -import jakarta.validation.constraints.NotNull; - -public class RaptorSecurityConfig -{ - private RaptorSecurity securitySystem = RaptorSecurity.ALLOW_ALL; - - @NotNull - public RaptorSecurity getSecuritySystem() - { - return securitySystem; - } - - @Config("raptor.security") - public RaptorSecurityConfig setSecuritySystem(RaptorSecurity securitySystem) - { - this.securitySystem = securitySystem; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityModule.java deleted file mode 100644 index b22d813660aa..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/security/RaptorSecurityModule.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import com.google.inject.Binder; -import com.google.inject.Module; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.base.security.ConnectorAccessControlModule; -import io.trino.plugin.base.security.FileBasedAccessControlModule; -import io.trino.plugin.base.security.ReadOnlySecurityModule; - -import static io.airlift.configuration.ConditionalModule.conditionalModule; -import static io.trino.plugin.raptor.legacy.security.RaptorSecurity.FILE; -import static io.trino.plugin.raptor.legacy.security.RaptorSecurity.READ_ONLY; - -public class RaptorSecurityModule - extends AbstractConfigurationAwareModule -{ - @Override - protected void setup(Binder binder) - { - install(new ConnectorAccessControlModule()); - bindSecurityModule(READ_ONLY, new ReadOnlySecurityModule()); - bindSecurityModule(FILE, new FileBasedAccessControlModule()); - } - - private void bindSecurityModule(RaptorSecurity raptorSecurity, Module module) - { - install(conditionalModule( - RaptorSecurityConfig.class, - security -> raptorSecurity.equals(security.getSecuritySystem()), - module)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BackupStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BackupStats.java deleted file mode 100644 index 8d6c25f19621..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BackupStats.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.errorprone.annotations.ThreadSafe; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import static io.trino.plugin.raptor.legacy.storage.ShardRecoveryManager.dataRate; - -@ThreadSafe -public class BackupStats -{ - private final DistributionStat copyToBackupBytesPerSecond = new DistributionStat(); - private final DistributionStat copyToBackupShardSizeBytes = new DistributionStat(); - private final DistributionStat copyToBackupTimeInMilliSeconds = new DistributionStat(); - private final DistributionStat queuedTimeMilliSeconds = new DistributionStat(); - - private final CounterStat backupSuccess = new CounterStat(); - private final CounterStat backupFailure = new CounterStat(); - private final CounterStat backupCorruption = new CounterStat(); - - public void addCopyShardDataRate(DataSize size, Duration duration) - { - DataSize rate = dataRate(size, duration).succinct(); - copyToBackupBytesPerSecond.add(rate.toBytes()); - copyToBackupShardSizeBytes.add(size.toBytes()); - copyToBackupTimeInMilliSeconds.add(duration.toMillis()); - } - - public void addQueuedTime(Duration queuedTime) - { - queuedTimeMilliSeconds.add(queuedTime.toMillis()); - } - - public void incrementBackupSuccess() - { - backupSuccess.update(1); - } - - public void incrementBackupFailure() - { - backupFailure.update(1); - } - - public void incrementBackupCorruption() - { - backupCorruption.update(1); - } - - @Managed - @Nested - public DistributionStat getCopyToBackupBytesPerSecond() - { - return copyToBackupBytesPerSecond; - } - - @Managed - @Nested - public DistributionStat getCopyToBackupShardSizeBytes() - { - return copyToBackupShardSizeBytes; - } - - @Managed - @Nested - public DistributionStat getCopyToBackupTimeInMilliSeconds() - { - return copyToBackupTimeInMilliSeconds; - } - - @Managed - @Nested - public DistributionStat getQueuedTimeMilliSeconds() - { - return queuedTimeMilliSeconds; - } - - @Managed - @Nested - public CounterStat getBackupSuccess() - { - return backupSuccess; - } - - @Managed - @Nested - public CounterStat getBackupFailure() - { - return backupFailure; - } - - @Managed - @Nested - public CounterStat getBackupCorruption() - { - return backupCorruption; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancer.java deleted file mode 100644 index 2193e4759b56..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancer.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.VerifyException; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.HashMultiset; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multiset; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.backup.BackupService; -import io.trino.plugin.raptor.legacy.metadata.BucketNode; -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.catalog.CatalogName; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; - -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static java.util.Comparator.comparingInt; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - -/** - * Service to balance buckets across active Raptor storage nodes in a cluster. - *

The objectives of this service are: - *

    - *
  1. For a given distribution, each node should be allocated the same number of buckets - * for a distribution. This enhances parallelism, and therefore query performance. - *
  2. The total disk utilization across the cluster should be balanced. This ensures that total - * cluster storage capacity is maximized. Simply allocating the same number of buckets - * to every node may not achieve this, as bucket sizes may vary dramatically across distributions. - *
- *

This prioritizes query performance over total cluster storage capacity, and therefore may - * produce a cluster state that is imbalanced in terms of disk utilization. - */ -public class BucketBalancer -{ - private static final Logger log = Logger.get(BucketBalancer.class); - - private final NodeSupplier nodeSupplier; - private final ShardManager shardManager; - private final boolean enabled; - private final Duration interval; - private final boolean backupAvailable; - private final boolean coordinator; - private final ScheduledExecutorService executor; - - private final AtomicBoolean started = new AtomicBoolean(); - - private final CounterStat bucketsBalanced = new CounterStat(); - private final CounterStat jobErrors = new CounterStat(); - - @Inject - public BucketBalancer( - NodeManager nodeManager, - NodeSupplier nodeSupplier, - ShardManager shardManager, - BucketBalancerConfig config, - BackupService backupService, - CatalogName catalogName) - { - this(nodeSupplier, - shardManager, - config.isBalancerEnabled(), - config.getBalancerInterval(), - backupService.isBackupAvailable(), - nodeManager.getCurrentNode().isCoordinator(), - catalogName.toString()); - } - - public BucketBalancer( - NodeSupplier nodeSupplier, - ShardManager shardManager, - boolean enabled, - Duration interval, - boolean backupAvailable, - boolean coordinator, - String catalogName) - { - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.enabled = enabled; - this.interval = requireNonNull(interval, "interval is null"); - this.backupAvailable = backupAvailable; - this.coordinator = coordinator; - this.executor = newSingleThreadScheduledExecutor(daemonThreadsNamed("bucket-balancer-" + catalogName)); - } - - @PostConstruct - public void start() - { - if (enabled && backupAvailable && coordinator && !started.getAndSet(true)) { - executor.scheduleWithFixedDelay(this::runBalanceJob, interval.toMillis(), interval.toMillis(), MILLISECONDS); - } - } - - @PreDestroy - public void shutdown() - { - executor.shutdownNow(); - } - - @Managed - @Nested - public CounterStat getBucketsBalanced() - { - return bucketsBalanced; - } - - @Managed - @Nested - public CounterStat getJobErrors() - { - return jobErrors; - } - - @Managed - public void startBalanceJob() - { - executor.submit(this::runBalanceJob); - } - - private void runBalanceJob() - { - try { - balance(); - } - catch (Throwable t) { - log.error(t, "Error balancing buckets"); - jobErrors.update(1); - } - } - - @VisibleForTesting - synchronized int balance() - { - log.info("Bucket balancer started. Computing assignments..."); - Multimap sourceToAssignmentChanges = computeAssignmentChanges(fetchClusterState()); - - log.info("Moving buckets..."); - int moves = updateAssignments(sourceToAssignmentChanges); - - log.info("Bucket balancing finished. Moved %s buckets.", moves); - return moves; - } - - private static Multimap computeAssignmentChanges(ClusterState clusterState) - { - Multimap sourceToAllocationChanges = HashMultimap.create(); - - Map allocationBytes = new HashMap<>(clusterState.getAssignedBytes()); - Set activeNodes = clusterState.getActiveNodes(); - - for (Distribution distribution : clusterState.getDistributionAssignments().keySet()) { - // number of buckets in this distribution assigned to a node - Multiset allocationCounts = HashMultiset.create(); - Collection distributionAssignments = clusterState.getDistributionAssignments().get(distribution); - distributionAssignments.stream() - .map(BucketAssignment::getNodeIdentifier) - .forEach(allocationCounts::add); - - int currentMin = allocationBytes.keySet().stream() - .mapToInt(allocationCounts::count) - .min() - .getAsInt(); - int currentMax = allocationBytes.keySet().stream() - .mapToInt(allocationCounts::count) - .max() - .getAsInt(); - - int numBuckets = distributionAssignments.size(); - int targetMin = (int) Math.floor((numBuckets * 1.0) / clusterState.getActiveNodes().size()); - int targetMax = (int) Math.ceil((numBuckets * 1.0) / clusterState.getActiveNodes().size()); - - log.info("Distribution %s: Current bucket skew: min %s, max %s. Target bucket skew: min %s, max %s", distribution.getId(), currentMin, currentMax, targetMin, targetMax); - - for (String source : ImmutableSet.copyOf(allocationCounts)) { - List existingAssignments = distributionAssignments.stream() - .filter(assignment -> assignment.getNodeIdentifier().equals(source)) - .collect(toList()); - - for (BucketAssignment existingAssignment : existingAssignments) { - if (activeNodes.contains(source) && allocationCounts.count(source) <= targetMin) { - break; - } - - // identify nodes with bucket counts lower than the computed target, and greedily select from this set based on projected disk utilization. - // greediness means that this may produce decidedly non-optimal results if one looks at the global distribution of buckets->nodes. - // also, this assumes that nodes in a cluster have identical storage capacity - String target = activeNodes.stream() - .filter(candidate -> !candidate.equals(source) && allocationCounts.count(candidate) < targetMax) - .sorted(comparingInt(allocationCounts::count)) - .min(Comparator.comparingDouble(allocationBytes::get)) - .orElseThrow(() -> new VerifyException("unable to find target for rebalancing")); - - long bucketSize = clusterState.getDistributionBucketSize().get(distribution); - - // only move bucket if it reduces imbalance - if (activeNodes.contains(source) && (allocationCounts.count(source) == targetMax && allocationCounts.count(target) == targetMin)) { - break; - } - - allocationCounts.remove(source); - allocationCounts.add(target); - allocationBytes.compute(source, (k, v) -> v - bucketSize); - allocationBytes.compute(target, (k, v) -> v + bucketSize); - - sourceToAllocationChanges.put( - existingAssignment.getNodeIdentifier(), - new BucketAssignment(existingAssignment.getDistributionId(), existingAssignment.getBucketNumber(), target)); - } - } - } - - return sourceToAllocationChanges; - } - - private int updateAssignments(Multimap sourceToAllocationChanges) - { - // perform moves in decreasing order of source node total assigned buckets - List sourceNodes = sourceToAllocationChanges.asMap().entrySet().stream() - .sorted((a, b) -> Integer.compare(b.getValue().size(), a.getValue().size())) - .map(Map.Entry::getKey) - .collect(toList()); - - int moves = 0; - for (String source : sourceNodes) { - for (BucketAssignment reassignment : sourceToAllocationChanges.get(source)) { - // todo: rate-limit new assignments - shardManager.updateBucketAssignment(reassignment.getDistributionId(), reassignment.getBucketNumber(), reassignment.getNodeIdentifier()); - bucketsBalanced.update(1); - moves++; - log.info("Distribution %s: Moved bucket %s from %s to %s", - reassignment.getDistributionId(), - reassignment.getBucketNumber(), - source, - reassignment.getNodeIdentifier()); - } - } - - return moves; - } - - @VisibleForTesting - ClusterState fetchClusterState() - { - Set activeNodes = nodeSupplier.getWorkerNodes().stream() - .map(Node::getNodeIdentifier) - .collect(toSet()); - - Map assignedNodeSize = new HashMap<>(activeNodes.stream().collect(toMap(node -> node, node -> 0L))); - ImmutableMultimap.Builder distributionAssignments = ImmutableMultimap.builder(); - ImmutableMap.Builder distributionBucketSize = ImmutableMap.builder(); - - for (Distribution distribution : shardManager.getDistributions()) { - long distributionSize = shardManager.getDistributionSizeInBytes(distribution.getId()); - long bucketSize = (long) (1.0 * distributionSize) / distribution.getBucketCount(); - distributionBucketSize.put(distribution, bucketSize); - - for (BucketNode bucketNode : shardManager.getBucketNodes(distribution.getId())) { - String node = bucketNode.getNodeIdentifier(); - distributionAssignments.put(distribution, new BucketAssignment(distribution.getId(), bucketNode.getBucketNumber(), node)); - assignedNodeSize.merge(node, bucketSize, Math::addExact); - } - } - - return new ClusterState(activeNodes, assignedNodeSize, distributionAssignments.build(), distributionBucketSize.buildOrThrow()); - } - - @VisibleForTesting - static class ClusterState - { - private final Set activeNodes; - private final Map assignedBytes; - private final Multimap distributionAssignments; - private final Map distributionBucketSize; - - public ClusterState( - Set activeNodes, - Map assignedBytes, - Multimap distributionAssignments, - Map distributionBucketSize) - { - this.activeNodes = ImmutableSet.copyOf(requireNonNull(activeNodes, "activeNodes is null")); - this.assignedBytes = ImmutableMap.copyOf(requireNonNull(assignedBytes, "assignedBytes is null")); - this.distributionAssignments = ImmutableMultimap.copyOf(requireNonNull(distributionAssignments, "distributionAssignments is null")); - this.distributionBucketSize = ImmutableMap.copyOf(requireNonNull(distributionBucketSize, "distributionBucketSize is null")); - } - - public Set getActiveNodes() - { - return activeNodes; - } - - public Map getAssignedBytes() - { - return assignedBytes; - } - - public Multimap getDistributionAssignments() - { - return distributionAssignments; - } - - public Map getDistributionBucketSize() - { - return distributionBucketSize; - } - } - - @VisibleForTesting - static class BucketAssignment - { - private final long distributionId; - private final int bucketNumber; - private final String nodeIdentifier; - - public BucketAssignment(long distributionId, int bucketNumber, String nodeIdentifier) - { - this.distributionId = distributionId; - this.bucketNumber = bucketNumber; - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - } - - public long getDistributionId() - { - return distributionId; - } - - public int getBucketNumber() - { - return bucketNumber; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancerConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancerConfig.java deleted file mode 100644 index 268d0ba1d237..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/BucketBalancerConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; - -import java.util.concurrent.TimeUnit; - -public class BucketBalancerConfig -{ - private boolean balancerEnabled = true; - private Duration balancerInterval = new Duration(6, TimeUnit.HOURS); - - public boolean isBalancerEnabled() - { - return balancerEnabled; - } - - @Config("storage.balancer-enabled") - public BucketBalancerConfig setBalancerEnabled(boolean balancerEnabled) - { - this.balancerEnabled = balancerEnabled; - return this; - } - - public Duration getBalancerInterval() - { - return balancerInterval; - } - - @Config("storage.balancer-interval") - @ConfigDescription("How often to run the global bucket balancer") - public BucketBalancerConfig setBalancerInterval(Duration balancerInterval) - { - this.balancerInterval = balancerInterval; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ColumnIndexStatsUtils.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ColumnIndexStatsUtils.java deleted file mode 100644 index 57a643b112ea..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ColumnIndexStatsUtils.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.sql.JDBCType; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; - -public final class ColumnIndexStatsUtils -{ - private ColumnIndexStatsUtils() {} - - public static JDBCType jdbcType(Type type) - { - if (type.equals(BOOLEAN)) { - return JDBCType.BOOLEAN; - } - if (type.equals(BIGINT) || type.equals(TIMESTAMP_MILLIS)) { - return JDBCType.BIGINT; - } - if (type.equals(INTEGER)) { - return JDBCType.INTEGER; - } - if (type.equals(DOUBLE)) { - return JDBCType.DOUBLE; - } - if (type.equals(DATE)) { - return JDBCType.INTEGER; - } - if (type instanceof VarcharType) { - return JDBCType.VARBINARY; - } - return null; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/FileStorageService.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/FileStorageService.java deleted file mode 100644 index 29e2fb6f0a1a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/FileStorageService.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.trino.spi.TrinoException; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.regex.Pattern; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.requireNonNull; - -public class FileStorageService - implements StorageService -{ - private static final Logger log = Logger.get(FileStorageService.class); - - private static final Pattern HEX_DIRECTORY = Pattern.compile("[0-9a-f]{2}"); - private static final String FILE_EXTENSION = ".orc"; - - private final File baseStorageDir; - private final File baseStagingDir; - private final File baseQuarantineDir; - - @Inject - public FileStorageService(StorageManagerConfig config) - { - this(config.getDataDirectory()); - } - - public FileStorageService(File dataDirectory) - { - File baseDataDir = requireNonNull(dataDirectory, "dataDirectory is null"); - this.baseStorageDir = new File(baseDataDir, "storage"); - this.baseStagingDir = new File(baseDataDir, "staging"); - this.baseQuarantineDir = new File(baseDataDir, "quarantine"); - } - - @Override - @PostConstruct - public void start() - { - deleteStagingFilesAsync(); - createParents(new File(baseStagingDir, ".")); - createParents(new File(baseStorageDir, ".")); - createParents(new File(baseQuarantineDir, ".")); - } - - @Override - public long getAvailableBytes() - { - return baseStorageDir.getUsableSpace(); - } - - @PreDestroy - public void stop() - throws IOException - { - deleteDirectory(baseStagingDir); - } - - @Override - public File getStorageFile(UUID shardUuid) - { - return getFileSystemPath(baseStorageDir, shardUuid); - } - - @Override - public File getStagingFile(UUID shardUuid) - { - String name = getFileSystemPath(new File("/"), shardUuid).getName(); - return new File(baseStagingDir, name); - } - - @Override - public File getQuarantineFile(UUID shardUuid) - { - String name = getFileSystemPath(new File("/"), shardUuid).getName(); - return new File(baseQuarantineDir, name); - } - - @Override - public Set getStorageShards() - { - ImmutableSet.Builder shards = ImmutableSet.builder(); - for (File level1 : listFiles(baseStorageDir, FileStorageService::isHexDirectory)) { - for (File level2 : listFiles(level1, FileStorageService::isHexDirectory)) { - for (File file : listFiles(level2, path -> true)) { - if (file.isFile()) { - uuidFromFileName(file.getName()).ifPresent(shards::add); - } - } - } - } - return shards.build(); - } - - @Override - public void createParents(File file) - { - File dir = file.getParentFile(); - if (!dir.mkdirs() && !dir.isDirectory()) { - throw new TrinoException(RAPTOR_ERROR, "Failed creating directories: " + dir); - } - } - - /** - * Generate a file system path for a shard UUID. - * This creates a three level deep directory structure where the first - * two levels each contain two hex digits (lowercase) of the UUID - * and the final level contains the full UUID. Example: - * 

-     * UUID: 701e1a79-74f7-4f56-b438-b41e8e7d019d
-     * Path: /base/70/1e/701e1a79-74f7-4f56-b438-b41e8e7d019d.orc
-     *
- * This ensures that files are spread out evenly through the tree - * while a path can still be easily navigated by a human being. - */ - public static File getFileSystemPath(File base, UUID shardUuid) - { - String uuid = shardUuid.toString().toLowerCase(ENGLISH); - return base.toPath() - .resolve(uuid.substring(0, 2)) - .resolve(uuid.substring(2, 4)) - .resolve(uuid + FILE_EXTENSION) - .toFile(); - } - - private void deleteStagingFilesAsync() - { - List files = listFiles(baseStagingDir, null); - if (!files.isEmpty()) { - new Thread(() -> { - for (File file : files) { - try { - Files.deleteIfExists(file.toPath()); - } - catch (IOException e) { - log.warn(e, "Failed to delete file: %s", file.getAbsolutePath()); - } - } - }, "background-staging-delete").start(); - } - } - - private static void deleteDirectory(File dir) - throws IOException - { - if (!dir.exists()) { - return; - } - File[] files = dir.listFiles(); - if (files == null) { - throw new IOException("Failed to list directory: " + dir); - } - for (File file : files) { - Files.deleteIfExists(file.toPath()); - } - Files.deleteIfExists(dir.toPath()); - } - - private static List listFiles(File dir, FileFilter filter) - { - File[] files = dir.listFiles(filter); - if (files == null) { - return ImmutableList.of(); - } - return ImmutableList.copyOf(files); - } - - private static boolean isHexDirectory(File file) - { - return file.isDirectory() && HEX_DIRECTORY.matcher(file.getName()).matches(); - } - - private static Optional uuidFromFileName(String name) - { - if (name.endsWith(FILE_EXTENSION)) { - name = name.substring(0, name.length() - FILE_EXTENSION.length()); - return uuidFromString(name); - } - return Optional.empty(); - } - - private static Optional uuidFromString(String value) - { - try { - return Optional.of(UUID.fromString(value)); - } - catch (IllegalArgumentException e) { - return Optional.empty(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileMetadata.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileMetadata.java deleted file mode 100644 index b96994e9ca7c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileMetadata.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableMap; -import io.trino.spi.type.TypeId; - -import java.util.Map; -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; - -public class OrcFileMetadata -{ - static final String KEY = "metadata"; - - private final Map columnTypes; - - @JsonCreator - public OrcFileMetadata(@JsonProperty("columnTypes") Map columnTypes) - { - this.columnTypes = ImmutableMap.copyOf(columnTypes); - } - - @JsonProperty - public Map getColumnTypes() - { - return columnTypes; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OrcFileMetadata that = (OrcFileMetadata) o; - return Objects.equals(columnTypes, that.columnTypes); - } - - @Override - public int hashCode() - { - return Objects.hash(columnTypes); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("columnTypes", columnTypes) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileRewriter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileRewriter.java deleted file mode 100644 index 3d154a6b6f0b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileRewriter.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.Maps; -import io.airlift.log.Logger; -import io.airlift.slice.Slice; -import io.trino.orc.FileOrcDataSource; -import io.trino.orc.OrcPredicate; -import io.trino.orc.OrcReader; -import io.trino.orc.OrcReaderOptions; -import io.trino.orc.OrcRecordReader; -import io.trino.orc.OrcWriter; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.DictionaryBlock; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import org.joda.time.DateTimeZone; - -import java.io.File; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.util.BitSet; -import java.util.List; -import java.util.Map; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static io.trino.orc.OrcReader.INITIAL_BATCH_SIZE; -import static io.trino.orc.OrcReader.createOrcReader; -import static io.trino.plugin.raptor.legacy.storage.OrcFileWriter.createOrcFileWriter; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.getColumnInfo; -import static java.lang.Math.toIntExact; - -public final class OrcFileRewriter -{ - private static final Logger log = Logger.get(OrcFileRewriter.class); - - private OrcFileRewriter() {} - - public static OrcFileInfo rewrite(TypeManager typeManager, File input, File output, BitSet rowsToDelete) - throws IOException - { - OrcReaderOptions options = new OrcReaderOptions(); - OrcReader reader = createOrcReader(new FileOrcDataSource(input, options), options) - .orElseThrow(() -> new IOException("File is empty: " + input)); - return rewrite(typeManager, reader, output, rowsToDelete); - } - - public static OrcFileInfo rewrite(TypeManager typeManager, OrcReader reader, File output, BitSet rowsToDelete) - throws IOException - { - long start = System.nanoTime(); - - List columnInfo = getColumnInfo(typeManager, reader); - - List columnNames = columnInfo.stream() - .map(info -> String.valueOf(info.getColumnId())) - .collect(toImmutableList()); - - List columnTypes = columnInfo.stream() - .map(ColumnInfo::getType) - .collect(toImmutableList()); - - OrcRecordReader recordReader = reader.createRecordReader( - reader.getRootColumn().getNestedColumns(), - columnTypes, - OrcPredicate.TRUE, - DateTimeZone.UTC, - newSimpleAggregatedMemoryContext(), - INITIAL_BATCH_SIZE, - RaptorPageSource::handleException); - - long fileRowCount = recordReader.getFileRowCount(); - if (fileRowCount < rowsToDelete.length()) { - throw new IOException("File has fewer rows than deletion vector"); - } - int deleteRowCount = rowsToDelete.cardinality(); - if (fileRowCount == deleteRowCount) { - return new OrcFileInfo(0, 0); - } - if (fileRowCount >= Integer.MAX_VALUE) { - throw new IOException("File has too many rows"); - } - - Map metadata = Maps.transformValues(reader.getFooter().getUserMetadata(), Slice::toStringUtf8); - - OrcFileInfo fileInfo; - try (OrcWriter writer = createOrcFileWriter(output, columnNames, columnTypes, reader.getFooter().getTypes(), metadata)) { - fileInfo = rewrite(recordReader, writer, rowsToDelete); - } - log.debug("Rewrote file in %s (input rows: %s, output rows: %s)", nanosSince(start), fileRowCount, fileRowCount - deleteRowCount); - return fileInfo; - } - - private static OrcFileInfo rewrite(OrcRecordReader reader, OrcWriter writer, BitSet rowsToDelete) - throws IOException - { - long rowCount = 0; - long uncompressedSize = 0; - - while (true) { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedIOException(); - } - - Page page = reader.nextPage(); - if (page == null) { - break; - } - - int start = toIntExact(reader.getFilePosition()); - page = maskedPage(page, rowsToDelete, start); - writer.write(page); - - rowCount += page.getPositionCount(); - uncompressedSize += page.getLogicalSizeInBytes(); - } - - return new OrcFileInfo(rowCount, uncompressedSize); - } - - private static Page maskedPage(Page page, BitSet rowsToDelete, int start) - { - int end = start + page.getPositionCount(); - if (rowsToDelete.nextSetBit(start) >= end) { - return page; - } - if (rowsToDelete.nextClearBit(start) >= end) { - return page.copyPositions(new int[0], 0, 0); - } - - int[] ids = new int[page.getPositionCount()]; - int size = 0; - for (int i = 0; i < ids.length; i++) { - if (!rowsToDelete.get(start + i)) { - ids[size] = i; - size++; - } - } - - Block[] maskedBlocks = new Block[page.getChannelCount()]; - for (int i = 0; i < maskedBlocks.length; i++) { - maskedBlocks[i] = DictionaryBlock.create(size, page.getBlock(i), ids); - } - return new Page(maskedBlocks); - } - - public static class OrcFileInfo - { - private final long rowCount; - private final long uncompressedSize; - - public OrcFileInfo(long rowCount, long uncompressedSize) - { - this.rowCount = rowCount; - this.uncompressedSize = uncompressedSize; - } - - public long getRowCount() - { - return rowCount; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileWriter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileWriter.java deleted file mode 100644 index 142e278bc01c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcFileWriter.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.trino.orc.OrcDataSink; -import io.trino.orc.OrcWriteValidation.OrcWriteValidationMode; -import io.trino.orc.OrcWriter; -import io.trino.orc.OrcWriterOptions; -import io.trino.orc.OrcWriterStats; -import io.trino.orc.OutputStreamOrcDataSink; -import io.trino.orc.metadata.ColumnMetadata; -import io.trino.orc.metadata.CompressionKind; -import io.trino.orc.metadata.OrcType; -import io.trino.plugin.raptor.legacy.util.SyncingFileOutputStream; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeManager; - -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.trino.orc.metadata.OrcType.createRootOrcType; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.toOrcFileType; - -public class OrcFileWriter - implements Closeable -{ - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - private final PageBuilder pageBuilder; - private final OrcWriter orcWriter; - - private boolean closed; - private long rowCount; - private long uncompressedSize; - - public OrcFileWriter(TypeManager typeManager, List columnIds, List columnTypes, File target) - { - checkArgument(columnIds.size() == columnTypes.size(), "ids and types mismatch"); - checkArgument(isUnique(columnIds), "ids must be unique"); - - List columnNames = columnIds.stream() - .map(Objects::toString) - .collect(toImmutableList()); - - columnTypes = columnTypes.stream() - .map(type -> toOrcFileType(type, typeManager)) - .collect(toImmutableList()); - - this.pageBuilder = new PageBuilder(columnTypes); - - ColumnMetadata orcTypes = createRootOrcType(columnNames, columnTypes); - Map metadata = createFileMetadata(columnIds, columnTypes); - - this.orcWriter = createOrcFileWriter(target, columnNames, columnTypes, orcTypes, metadata); - } - - public void appendPages(List pages) - { - for (Page page : pages) { - appendPage(page); - } - } - - public void appendPages(List pages, int[] pageIndexes, int[] positionIndexes) - { - checkArgument(pageIndexes.length == positionIndexes.length, "pageIndexes and positionIndexes do not match"); - - for (int i = 0; i < pageIndexes.length; i++) { - Page page = pages.get(pageIndexes[i]); - int position = positionIndexes[i]; - - pageBuilder.declarePosition(); - for (int channel = 0; channel < page.getChannelCount(); channel++) { - Block block = page.getBlock(channel); - BlockBuilder output = pageBuilder.getBlockBuilder(channel); - pageBuilder.getType(channel).appendTo(block, position, output); - } - - if (pageBuilder.isFull()) { - appendPage(pageBuilder.build()); - pageBuilder.reset(); - } - } - - if (!pageBuilder.isEmpty()) { - appendPage(pageBuilder.build()); - pageBuilder.reset(); - } - } - - private void appendPage(Page page) - { - rowCount += page.getPositionCount(); - uncompressedSize += page.getLogicalSizeInBytes(); - - try { - orcWriter.write(page); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to write data", e); - } - } - - public long getRowCount() - { - return rowCount; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - - @Override - public void close() - { - if (closed) { - return; - } - closed = true; - - try { - orcWriter.close(); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to close writer", e); - } - } - - private static boolean isUnique(Collection items) - { - return new HashSet<>(items).size() == items.size(); - } - - private static Map createFileMetadata(List columnIds, List columnTypes) - { - ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); - for (int i = 0; i < columnIds.size(); i++) { - columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeId()); - } - OrcFileMetadata metadata = new OrcFileMetadata(columnTypesMap.buildOrThrow()); - return ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(metadata)); - } - - private static OrcDataSink createOrcDataSink(File target) - { - try { - return OutputStreamOrcDataSink.create(new SyncingFileOutputStream(target)); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to open output file: " + target, e); - } - } - - public static OrcWriter createOrcFileWriter( - File target, - List columnNames, - List types, - ColumnMetadata orcTypes, - Map metadata) - { - return new OrcWriter( - createOrcDataSink(target), - columnNames, - types, - orcTypes, - CompressionKind.SNAPPY, - new OrcWriterOptions(), - metadata, - true, - OrcWriteValidationMode.BOTH, - new OrcWriterStats()); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java deleted file mode 100644 index bbc7aba4661d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.memory.context.AggregatedMemoryContext; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcRecordReader; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.block.RowBlock; -import io.trino.spi.block.RunLengthEncodedBlock; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.type.Type; -import io.trino.spi.type.UuidType; - -import java.io.IOException; -import java.util.List; -import java.util.OptionalInt; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.base.util.Closables.closeAllSuppress; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.UuidType.javaUuidToTrinoUuid; -import static java.util.Objects.requireNonNull; - -public class RaptorPageSource - implements ConnectorPageSource -{ - private final OrcRecordReader recordReader; - private final List columnAdaptations; - private final OrcDataSource orcDataSource; - - private final AggregatedMemoryContext memoryContext; - - private boolean closed; - - public RaptorPageSource( - OrcRecordReader recordReader, - List columnAdaptations, - OrcDataSource orcDataSource, - AggregatedMemoryContext memoryContext) - { - this.recordReader = requireNonNull(recordReader, "recordReader is null"); - this.columnAdaptations = ImmutableList.copyOf(requireNonNull(columnAdaptations, "columnAdaptations is null")); - this.orcDataSource = requireNonNull(orcDataSource, "orcDataSource is null"); - - this.memoryContext = requireNonNull(memoryContext, "memoryContext is null"); - } - - @Override - public long getCompletedBytes() - { - return orcDataSource.getReadBytes(); - } - - @Override - public long getReadTimeNanos() - { - return orcDataSource.getReadTimeNanos(); - } - - @Override - public boolean isFinished() - { - return closed; - } - - @Override - public Page getNextPage() - { - Page page; - try { - page = recordReader.nextPage(); - } - catch (IOException | RuntimeException e) { - closeAllSuppress(e, this); - throw handleException(e); - } - - if (page == null) { - close(); - return null; - } - - long filePosition = recordReader.getFilePosition(); - Block[] blocks = new Block[columnAdaptations.size()]; - for (int i = 0; i < columnAdaptations.size(); i++) { - blocks[i] = columnAdaptations.get(i).block(page, filePosition); - } - return new Page(page.getPositionCount(), blocks); - } - - public static TrinoException handleException(Exception exception) - { - if (exception instanceof TrinoException) { - return (TrinoException) exception; - } - throw new TrinoException(RAPTOR_ERROR, exception); - } - - @Override - public void close() - { - if (closed) { - return; - } - closed = true; - - try { - recordReader.close(); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, e); - } - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("columns", columnAdaptations) - .toString(); - } - - @Override - public long getMemoryUsage() - { - return memoryContext.getBytes(); - } - - public interface ColumnAdaptation - { - Block block(Page sourcePage, long filePosition); - - static ColumnAdaptation nullColumn(Type type) - { - return new NullColumn(type); - } - - static ColumnAdaptation shardUuidColumn(UUID shardUuid) - { - return new ShardUuidAdaptation(shardUuid); - } - - static ColumnAdaptation bucketNumberColumn(OptionalInt bucketNumber) - { - if (bucketNumber.isEmpty()) { - return nullColumn(INTEGER); - } - return new BucketNumberColumn(bucketNumber.getAsInt()); - } - - static ColumnAdaptation rowIdColumn() - { - return RowIdColumn.INSTANCE; - } - - static ColumnAdaptation mergeRowIdColumn(OptionalInt bucketNumber, UUID shardUuid) - { - return new MergeRowIdColumn(bucketNumber, shardUuid); - } - - static ColumnAdaptation sourceColumn(int index) - { - return new SourceColumn(index); - } - } - - private static class ShardUuidAdaptation - implements ColumnAdaptation - { - private final Block shardUuidBlock; - - public ShardUuidAdaptation(UUID shardUuid) - { - Slice slice = utf8Slice(shardUuid.toString()); - BlockBuilder blockBuilder = SHARD_UUID_COLUMN_TYPE.createBlockBuilder(null, 1, slice.length()); - SHARD_UUID_COLUMN_TYPE.writeSlice(blockBuilder, slice); - this.shardUuidBlock = blockBuilder.build(); - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - return RunLengthEncodedBlock.create(shardUuidBlock, sourcePage.getPositionCount()); - } - - @Override - public String toString() - { - return toStringHelper(this) - .toString(); - } - } - - private static class RowIdColumn - implements ColumnAdaptation - { - public static final RowIdColumn INSTANCE = new RowIdColumn(); - - @Override - public Block block(Page sourcePage, long filePosition) - { - int count = sourcePage.getPositionCount(); - BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(count); - for (int i = 0; i < count; i++) { - BIGINT.writeLong(builder, filePosition + i); - } - return builder.build(); - } - - @Override - public String toString() - { - return toStringHelper(this) - .toString(); - } - } - - private static class MergeRowIdColumn - implements ColumnAdaptation - { - private final Block bucketNumberValue; - private final Block shardUuidValue; - - public MergeRowIdColumn(OptionalInt bucketNumber, UUID shardUuid) - { - BlockBuilder blockBuilder = INTEGER.createFixedSizeBlockBuilder(1); - bucketNumber.ifPresentOrElse(value -> INTEGER.writeLong(blockBuilder, value), blockBuilder::appendNull); - bucketNumberValue = blockBuilder.build(); - - BlockBuilder builder = UuidType.UUID.createFixedSizeBlockBuilder(1); - UuidType.UUID.writeSlice(builder, javaUuidToTrinoUuid(shardUuid)); - shardUuidValue = builder.build(); - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - Block bucketNumberBlock = RunLengthEncodedBlock.create(bucketNumberValue, sourcePage.getPositionCount()); - Block shardUuidBlock = RunLengthEncodedBlock.create(shardUuidValue, sourcePage.getPositionCount()); - Block rowIdBlock = RowIdColumn.INSTANCE.block(sourcePage, filePosition); - return RowBlock.fromFieldBlocks( - sourcePage.getPositionCount(), - new Block[] {bucketNumberBlock, shardUuidBlock, rowIdBlock}); - } - } - - private static class NullColumn - implements ColumnAdaptation - { - private final Type type; - private final Block nullBlock; - - public NullColumn(Type type) - { - this.type = requireNonNull(type, "type is null"); - this.nullBlock = type.createBlockBuilder(null, 1, 0) - .appendNull() - .build(); - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - return RunLengthEncodedBlock.create(nullBlock, sourcePage.getPositionCount()); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("type", type) - .toString(); - } - } - - private static class BucketNumberColumn - implements ColumnAdaptation - { - private final Block bucketNumberBlock; - - public BucketNumberColumn(int bucketNumber) - { - BlockBuilder blockBuilder = INTEGER.createFixedSizeBlockBuilder(1); - INTEGER.writeLong(blockBuilder, bucketNumber); - this.bucketNumberBlock = blockBuilder.build(); - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - return RunLengthEncodedBlock.create(bucketNumberBlock, sourcePage.getPositionCount()); - } - - @Override - public String toString() - { - return toStringHelper(this) - .toString(); - } - } - - private static class SourceColumn - implements ColumnAdaptation - { - private final int index; - - public SourceColumn(int index) - { - checkArgument(index >= 0, "index is negative"); - this.index = index; - } - - @Override - public Block block(Page sourcePage, long filePosition) - { - return sourcePage.getBlock(index); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("index", index) - .toString(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java deleted file mode 100644 index 16710298979e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java +++ /dev/null @@ -1,668 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.airlift.json.JsonCodec; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.airlift.slice.XxHash64; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.memory.context.AggregatedMemoryContext; -import io.trino.orc.FileOrcDataSource; -import io.trino.orc.OrcColumn; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcPredicate; -import io.trino.orc.OrcReader; -import io.trino.orc.OrcReaderOptions; -import io.trino.orc.OrcRecordReader; -import io.trino.orc.TupleDomainOrcPredicate; -import io.trino.orc.TupleDomainOrcPredicate.TupleDomainOrcPredicateBuilder; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.backup.BackupManager; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.ColumnStats; -import io.trino.plugin.raptor.legacy.metadata.ShardDelta; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; -import io.trino.plugin.raptor.legacy.storage.OrcFileRewriter.OrcFileInfo; -import io.trino.plugin.raptor.legacy.storage.RaptorPageSource.ColumnAdaptation; -import io.trino.spi.NodeManager; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.MapType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeSignature; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import jakarta.annotation.PreDestroy; - -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Throwables.throwIfInstanceOf; -import static com.google.common.collect.Maps.uniqueIndex; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.units.DataSize.Unit.PETABYTE; -import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static io.trino.orc.OrcReader.INITIAL_BATCH_SIZE; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isBucketNumberColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isHiddenColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isMergeRowIdColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isShardRowIdColumn; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.isShardUuidColumn; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_LOCAL_DISK_FULL; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_TIMEOUT; -import static io.trino.plugin.raptor.legacy.storage.ShardStats.computeColumnStats; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.TypeSignatureParameter.typeParameter; -import static java.lang.Math.min; -import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.completedFuture; -import static java.util.concurrent.CompletableFuture.supplyAsync; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static java.util.stream.Collectors.toList; -import static org.joda.time.DateTimeZone.UTC; - -public class RaptorStorageManager - implements StorageManager -{ - private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); - - private static final long MAX_ROWS = 1_000_000_000; - // TODO: do not limit the max size of blocks to read for now; enable the limit when the Hive connector is ready - private static final DataSize HUGE_MAX_READ_BLOCK_SIZE = DataSize.of(1, PETABYTE); - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - private final String nodeId; - private final StorageService storageService; - private final Optional backupStore; - private final OrcReaderOptions orcReaderOptions; - private final BackupManager backupManager; - private final ShardRecoveryManager recoveryManager; - private final ShardRecorder shardRecorder; - private final Duration recoveryTimeout; - private final long maxShardRows; - private final DataSize maxShardSize; - private final DataSize minAvailableSpace; - private final TypeManager typeManager; - private final ExecutorService deletionExecutor; - private final ExecutorService commitExecutor; - - @Inject - public RaptorStorageManager( - NodeManager nodeManager, - StorageService storageService, - Optional backupStore, - StorageManagerConfig config, - CatalogName catalogName, - BackupManager backgroundBackupManager, - ShardRecoveryManager recoveryManager, - ShardRecorder shardRecorder, - TypeManager typeManager) - { - this(nodeManager.getCurrentNode().getNodeIdentifier(), - storageService, - backupStore, - config.toOrcReaderOptions(), - backgroundBackupManager, - recoveryManager, - shardRecorder, - typeManager, - catalogName.toString(), - config.getDeletionThreads(), - config.getShardRecoveryTimeout(), - config.getMaxShardRows(), - config.getMaxShardSize(), - config.getMinAvailableSpace()); - } - - public RaptorStorageManager( - String nodeId, - StorageService storageService, - Optional backupStore, - OrcReaderOptions orcReaderOptions, - BackupManager backgroundBackupManager, - ShardRecoveryManager recoveryManager, - ShardRecorder shardRecorder, - TypeManager typeManager, - String connectorId, - int deletionThreads, - Duration shardRecoveryTimeout, - long maxShardRows, - DataSize maxShardSize, - DataSize minAvailableSpace) - { - this.nodeId = requireNonNull(nodeId, "nodeId is null"); - this.storageService = requireNonNull(storageService, "storageService is null"); - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.orcReaderOptions = orcReaderOptions - .withMaxReadBlockSize(HUGE_MAX_READ_BLOCK_SIZE); - - backupManager = requireNonNull(backgroundBackupManager, "backgroundBackupManager is null"); - this.recoveryManager = requireNonNull(recoveryManager, "recoveryManager is null"); - this.recoveryTimeout = requireNonNull(shardRecoveryTimeout, "shardRecoveryTimeout is null"); - - checkArgument(maxShardRows > 0, "maxShardRows must be > 0"); - this.maxShardRows = min(maxShardRows, MAX_ROWS); - this.maxShardSize = requireNonNull(maxShardSize, "maxShardSize is null"); - this.minAvailableSpace = requireNonNull(minAvailableSpace, "minAvailableSpace is null"); - this.shardRecorder = requireNonNull(shardRecorder, "shardRecorder is null"); - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - this.deletionExecutor = newFixedThreadPool(deletionThreads, daemonThreadsNamed("raptor-delete-" + connectorId + "-%s")); - this.commitExecutor = newCachedThreadPool(daemonThreadsNamed("raptor-commit-" + connectorId + "-%s")); - } - - @PreDestroy - public void shutdown() - { - deletionExecutor.shutdownNow(); - commitExecutor.shutdown(); - } - - @Override - public ConnectorPageSource getPageSource( - UUID shardUuid, - OptionalInt bucketNumber, - List columnIds, - List columnTypes, - TupleDomain effectivePredicate, - OrcReaderOptions orcReaderOptions) - { - orcReaderOptions = orcReaderOptions.withMaxReadBlockSize(HUGE_MAX_READ_BLOCK_SIZE); - OrcDataSource dataSource = openShard(shardUuid, orcReaderOptions); - - AggregatedMemoryContext memoryUsage = newSimpleAggregatedMemoryContext(); - - try { - OrcReader reader = OrcReader.createOrcReader(dataSource, orcReaderOptions) - .orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "Data file is empty for shard " + shardUuid)); - - Map indexMap = columnIdIndex(reader.getRootColumn().getNestedColumns()); - List fileReadColumn = new ArrayList<>(columnIds.size()); - List fileReadTypes = new ArrayList<>(columnIds.size()); - List columnAdaptations = new ArrayList<>(columnIds.size()); - for (int i = 0; i < columnIds.size(); i++) { - long columnId = columnIds.get(i); - - if (isHiddenColumn(columnId)) { - columnAdaptations.add(specialColumnAdaptation(columnId, shardUuid, bucketNumber)); - continue; - } - - Type type = toOrcFileType(columnTypes.get(i), typeManager); - OrcColumn fileColumn = indexMap.get(columnId); - if (fileColumn == null) { - columnAdaptations.add(ColumnAdaptation.nullColumn(type)); - } - else { - int sourceIndex = fileReadColumn.size(); - columnAdaptations.add(ColumnAdaptation.sourceColumn(sourceIndex)); - fileReadColumn.add(fileColumn); - fileReadTypes.add(type); - } - } - - OrcPredicate predicate = getPredicate(effectivePredicate, indexMap); - - OrcRecordReader recordReader = reader.createRecordReader( - fileReadColumn, - fileReadTypes, - predicate, - UTC, - memoryUsage, - INITIAL_BATCH_SIZE, - RaptorPageSource::handleException); - - return new RaptorPageSource(recordReader, columnAdaptations, dataSource, memoryUsage); - } - catch (IOException | RuntimeException e) { - closeQuietly(dataSource); - throw new TrinoException(RAPTOR_ERROR, "Failed to create page source for shard " + shardUuid, e); - } - catch (Throwable t) { - closeQuietly(dataSource); - throw t; - } - } - - private static ColumnAdaptation specialColumnAdaptation(long columnId, UUID shardUuid, OptionalInt bucketNumber) - { - if (isShardRowIdColumn(columnId)) { - return ColumnAdaptation.rowIdColumn(); - } - if (isShardUuidColumn(columnId)) { - return ColumnAdaptation.shardUuidColumn(shardUuid); - } - if (isBucketNumberColumn(columnId)) { - return ColumnAdaptation.bucketNumberColumn(bucketNumber); - } - if (isMergeRowIdColumn(columnId)) { - return ColumnAdaptation.mergeRowIdColumn(bucketNumber, shardUuid); - } - throw new TrinoException(RAPTOR_ERROR, "Invalid column ID: " + columnId); - } - - @Override - public StoragePageSink createStoragePageSink(long transactionId, OptionalInt bucketNumber, List columnIds, List columnTypes, boolean checkSpace) - { - if (checkSpace && storageService.getAvailableBytes() < minAvailableSpace.toBytes()) { - throw new TrinoException(RAPTOR_LOCAL_DISK_FULL, "Local disk is full on node " + nodeId); - } - return new RaptorStoragePageSink(transactionId, columnIds, columnTypes, bucketNumber); - } - - @Override - public ShardRewriter createShardRewriter(long transactionId, OptionalInt bucketNumber, UUID shardUuid) - { - return rowsToDelete -> { - if (rowsToDelete.isEmpty()) { - return completedFuture(ImmutableList.of()); - } - return supplyAsync(() -> rewriteShard(transactionId, bucketNumber, shardUuid, rowsToDelete), deletionExecutor); - }; - } - - private void writeShard(UUID shardUuid) - { - if (backupStore.isPresent() && !backupStore.get().shardExists(shardUuid)) { - throw new TrinoException(RAPTOR_ERROR, "Backup does not exist after write"); - } - - File stagingFile = storageService.getStagingFile(shardUuid); - File storageFile = storageService.getStorageFile(shardUuid); - - storageService.createParents(storageFile); - - try { - Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to move shard file", e); - } - } - - @VisibleForTesting - OrcDataSource openShard(UUID shardUuid, OrcReaderOptions orcReaderOptions) - { - File file = storageService.getStorageFile(shardUuid).getAbsoluteFile(); - - if (!file.exists() && backupStore.isPresent()) { - try { - Future future = recoveryManager.recoverShard(shardUuid); - future.get(recoveryTimeout.toMillis(), TimeUnit.MILLISECONDS); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - catch (ExecutionException e) { - if (e.getCause() != null) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - } - throw new TrinoException(RAPTOR_RECOVERY_ERROR, "Error recovering shard " + shardUuid, e.getCause()); - } - catch (TimeoutException e) { - throw new TrinoException(RAPTOR_RECOVERY_TIMEOUT, "Shard is being recovered from backup. Please retry in a few minutes: " + shardUuid); - } - } - - try { - return fileOrcDataSource(orcReaderOptions, file); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to open shard file: " + file, e); - } - } - - private static FileOrcDataSource fileOrcDataSource(OrcReaderOptions orcReaderOptions, File file) - throws FileNotFoundException - { - return new FileOrcDataSource(file, orcReaderOptions); - } - - private ShardInfo createShardInfo(UUID shardUuid, OptionalInt bucketNumber, File file, Set nodes, long rowCount, long uncompressedSize) - { - return new ShardInfo(shardUuid, bucketNumber, nodes, computeShardStats(file), rowCount, file.length(), uncompressedSize, xxhash64(file)); - } - - private List computeShardStats(File file) - { - try (OrcDataSource dataSource = fileOrcDataSource(orcReaderOptions, file)) { - OrcReader reader = OrcReader.createOrcReader(dataSource, orcReaderOptions) - .orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "Data file is empty: " + file)); - - ImmutableList.Builder list = ImmutableList.builder(); - for (ColumnInfo info : getColumnInfo(typeManager, reader)) { - computeColumnStats(reader, info.getColumnId(), info.getType(), typeManager).ifPresent(list::add); - } - return list.build(); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to read file: " + file, e); - } - } - - @VisibleForTesting - Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, BitSet rowsToDelete) - { - if (rowsToDelete.isEmpty()) { - return ImmutableList.of(); - } - - UUID newShardUuid = UUID.randomUUID(); - File input = storageService.getStorageFile(shardUuid); - File output = storageService.getStagingFile(newShardUuid); - - OrcFileInfo info = rewriteFile(typeManager, input, output, rowsToDelete); - long rowCount = info.getRowCount(); - - if (rowCount == 0) { - return shardDelta(shardUuid, Optional.empty()); - } - - shardRecorder.recordCreatedShard(transactionId, newShardUuid); - - // submit for backup and wait until it finishes - getFutureValue(backupManager.submit(newShardUuid, output)); - - Set nodes = ImmutableSet.of(nodeId); - long uncompressedSize = info.getUncompressedSize(); - - ShardInfo shard = createShardInfo(newShardUuid, bucketNumber, output, nodes, rowCount, uncompressedSize); - - writeShard(newShardUuid); - - return shardDelta(shardUuid, Optional.of(shard)); - } - - private static Collection shardDelta(UUID oldShardUuid, Optional shardInfo) - { - List newShards = shardInfo.map(ImmutableList::of).orElse(ImmutableList.of()); - ShardDelta delta = new ShardDelta(ImmutableList.of(oldShardUuid), newShards); - return ImmutableList.of(Slices.wrappedBuffer(SHARD_DELTA_CODEC.toJsonBytes(delta))); - } - - private static OrcFileInfo rewriteFile(TypeManager typeManager, File input, File output, BitSet rowsToDelete) - { - try { - return OrcFileRewriter.rewrite(typeManager, input, output, rowsToDelete); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to rewrite shard file: " + input, e); - } - } - - static List getColumnInfo(TypeManager typeManager, OrcReader reader) - { - return getColumnInfoFromOrcUserMetadata(typeManager, getOrcFileMetadata(reader)); - } - - static long xxhash64(File file) - { - try (InputStream in = new FileInputStream(file)) { - return XxHash64.hash(in); - } - catch (IOException e) { - throw new TrinoException(RAPTOR_ERROR, "Failed to read file: " + file, e); - } - } - - private static OrcFileMetadata getOrcFileMetadata(OrcReader reader) - { - return Optional.ofNullable(reader.getFooter().getUserMetadata().get(OrcFileMetadata.KEY)) - .map(slice -> METADATA_CODEC.fromJson(slice.getBytes())) - .orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "No Raptor metadata in file")); - } - - private static List getColumnInfoFromOrcUserMetadata(TypeManager typeManager, OrcFileMetadata orcFileMetadata) - { - return orcFileMetadata.getColumnTypes().entrySet() - .stream() - .sorted(Map.Entry.comparingByKey()) - .map(entry -> new ColumnInfo(entry.getKey(), typeManager.getType(entry.getValue()))) - .collect(toList()); - } - - static Type toOrcFileType(Type raptorType, TypeManager typeManager) - { - if ((raptorType == BOOLEAN) || - (raptorType == TINYINT) || - (raptorType == SMALLINT) || - (raptorType == INTEGER) || - (raptorType == BIGINT) || - (raptorType == REAL) || - (raptorType == DOUBLE) || - (raptorType == DATE) || - (raptorType instanceof DecimalType) || - (raptorType instanceof VarcharType) || - (raptorType instanceof VarbinaryType)) { - return raptorType; - } - if (raptorType.equals(TIMESTAMP_MILLIS)) { - // TIMESTAMPS are stored as BIGINT to avoid the poor encoding in ORC - return BIGINT; - } - if (raptorType instanceof ArrayType) { - Type elementType = toOrcFileType(((ArrayType) raptorType).getElementType(), typeManager); - return new ArrayType(elementType); - } - if (raptorType instanceof MapType) { - TypeSignature keyType = toOrcFileType(((MapType) raptorType).getKeyType(), typeManager).getTypeSignature(); - TypeSignature valueType = toOrcFileType(((MapType) raptorType).getValueType(), typeManager).getTypeSignature(); - return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(typeParameter(keyType), typeParameter(valueType))); - } - throw new TrinoException(NOT_SUPPORTED, "Unsupported type: " + raptorType); - } - - private static OrcPredicate getPredicate(TupleDomain effectivePredicate, Map indexMap) - { - TupleDomainOrcPredicateBuilder predicateBuilder = TupleDomainOrcPredicate.builder(); - effectivePredicate.getDomains().get().forEach((columnHandle, value) -> { - OrcColumn fileColumn = indexMap.get(columnHandle.getColumnId()); - if (fileColumn != null) { - predicateBuilder.addColumn(fileColumn.getColumnId(), value); - } - }); - return predicateBuilder.build(); - } - - private static Map columnIdIndex(List columns) - { - return uniqueIndex(columns, column -> Long.valueOf(column.getColumnName())); - } - - private class RaptorStoragePageSink - implements StoragePageSink - { - private final long transactionId; - private final List columnIds; - private final List columnTypes; - private final OptionalInt bucketNumber; - - private final List stagingFiles = new ArrayList<>(); - private final List shards = new ArrayList<>(); - private final List> futures = new ArrayList<>(); - - private boolean committed; - private OrcFileWriter writer; - private UUID shardUuid; - - public RaptorStoragePageSink(long transactionId, List columnIds, List columnTypes, OptionalInt bucketNumber) - { - this.transactionId = transactionId; - this.columnIds = ImmutableList.copyOf(requireNonNull(columnIds, "columnIds is null")); - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - } - - @Override - public void appendPages(List pages) - { - createWriterIfNecessary(); - writer.appendPages(pages); - } - - @Override - public void appendPages(List inputPages, int[] pageIndexes, int[] positionIndexes) - { - createWriterIfNecessary(); - writer.appendPages(inputPages, pageIndexes, positionIndexes); - } - - @Override - public boolean isFull() - { - if (writer == null) { - return false; - } - return (writer.getRowCount() >= maxShardRows) || (writer.getUncompressedSize() >= maxShardSize.toBytes()); - } - - @Override - public void flush() - { - if (writer != null) { - writer.close(); - - shardRecorder.recordCreatedShard(transactionId, shardUuid); - - File stagingFile = storageService.getStagingFile(shardUuid); - futures.add(backupManager.submit(shardUuid, stagingFile)); - - Set nodes = ImmutableSet.of(nodeId); - long rowCount = writer.getRowCount(); - long uncompressedSize = writer.getUncompressedSize(); - - shards.add(createShardInfo(shardUuid, bucketNumber, stagingFile, nodes, rowCount, uncompressedSize)); - - writer = null; - shardUuid = null; - } - } - - @Override - public CompletableFuture> commit() - { - checkState(!committed, "already committed"); - committed = true; - - flush(); - - return allAsList(futures).thenApplyAsync(_ -> { - for (ShardInfo shard : shards) { - writeShard(shard.getShardUuid()); - } - return ImmutableList.copyOf(shards); - }, commitExecutor); - } - - @SuppressWarnings("ResultOfMethodCallIgnored") - @Override - public void rollback() - { - try { - if (writer != null) { - writer.close(); - writer = null; - } - } - finally { - for (File file : stagingFiles) { - file.delete(); - } - - // cancel incomplete backup jobs - futures.forEach(future -> future.cancel(true)); - - // delete completed backup shards - backupStore.ifPresent(backupStore -> { - for (ShardInfo shard : shards) { - backupStore.deleteShard(shard.getShardUuid()); - } - }); - } - } - - private void createWriterIfNecessary() - { - if (writer == null) { - shardUuid = UUID.randomUUID(); - File stagingFile = storageService.getStagingFile(shardUuid); - storageService.createParents(stagingFile); - stagingFiles.add(stagingFile); - writer = new OrcFileWriter(typeManager, columnIds, columnTypes, stagingFile); - } - } - } - - private static void closeQuietly(Closeable closeable) - { - try { - closeable.close(); - } - catch (IOException _) { - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardEjector.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardEjector.java deleted file mode 100644 index e9d4c819ab34..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardEjector.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.catalog.CatalogName; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.io.File; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Queue; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Maps.filterKeys; -import static com.google.common.collect.Maps.filterValues; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static java.lang.Math.round; -import static java.util.Comparator.comparingLong; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toCollection; -import static java.util.stream.Collectors.toSet; - -public class ShardEjector -{ - private static final Logger log = Logger.get(ShardEjector.class); - - private final String currentNode; - private final NodeSupplier nodeSupplier; - private final ShardManager shardManager; - private final StorageService storageService; - private final Duration interval; - private final Optional backupStore; - private final ScheduledExecutorService executor; - - private final AtomicBoolean started = new AtomicBoolean(); - - private final CounterStat shardsEjected = new CounterStat(); - private final CounterStat jobErrors = new CounterStat(); - - @Inject - public ShardEjector( - NodeManager nodeManager, - NodeSupplier nodeSupplier, - ShardManager shardManager, - StorageService storageService, - StorageManagerConfig config, - Optional backupStore, - CatalogName catalogName) - { - this(nodeManager.getCurrentNode().getNodeIdentifier(), - nodeSupplier, - shardManager, - storageService, - config.getShardEjectorInterval(), - backupStore, - catalogName.toString()); - } - - public ShardEjector( - String currentNode, - NodeSupplier nodeSupplier, - ShardManager shardManager, - StorageService storageService, - Duration interval, - Optional backupStore, - String connectorId) - { - this.currentNode = requireNonNull(currentNode, "currentNode is null"); - this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.storageService = requireNonNull(storageService, "storageService is null"); - this.interval = requireNonNull(interval, "interval is null"); - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.executor = newScheduledThreadPool(1, daemonThreadsNamed("shard-ejector-" + connectorId)); - } - - @PostConstruct - public void start() - { - if (backupStore.isEmpty()) { - return; - } - if (!started.getAndSet(true)) { - startJob(); - } - } - - @PreDestroy - public void shutdown() - { - executor.shutdownNow(); - } - - @Managed - @Nested - public CounterStat getShardsEjected() - { - return shardsEjected; - } - - @Managed - @Nested - public CounterStat getJobErrors() - { - return jobErrors; - } - - private void startJob() - { - executor.scheduleWithFixedDelay(() -> { - try { - // jitter to avoid overloading database - long interval = this.interval.roundTo(SECONDS); - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval)); - process(); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable t) { - log.error(t, "Error ejecting shards"); - jobErrors.update(1); - } - }, 0, interval.toMillis(), MILLISECONDS); - } - - @VisibleForTesting - void process() - { - checkState(backupStore.isPresent(), "backup store must be present"); - - // get the size of assigned shards for each node - Map nodes = shardManager.getNodeBytes(); - - Set activeNodes = nodeSupplier.getWorkerNodes().stream() - .map(Node::getNodeIdentifier) - .collect(toSet()); - - // only include active nodes - nodes = new HashMap<>(filterKeys(nodes, activeNodes::contains)); - - if (nodes.isEmpty()) { - return; - } - - // get current node size - if (!nodes.containsKey(currentNode)) { - return; - } - long nodeSize = nodes.get(currentNode); - - // get average node size - long averageSize = round(nodes.values().stream() - .mapToLong(Long::longValue) - .average() - .getAsDouble()); - long maxSize = round(averageSize * 1.01); - - // skip if not above max - if (nodeSize <= maxSize) { - return; - } - - // only include nodes that are below threshold - nodes = new HashMap<>(filterValues(nodes, size -> size <= averageSize)); - - // get non-bucketed node shards by size, largest to smallest - Queue queue = shardManager.getNodeShards(currentNode).stream() - .filter(shard -> shard.getBucketNumber().isEmpty()) - .sorted(comparingLong(ShardMetadata::getCompressedSize).reversed()) - // eject shards while current node is above max - .collect(toCollection(ArrayDeque::new)); - while ((nodeSize > maxSize) && !queue.isEmpty()) { - ShardMetadata shard = queue.remove(); - long shardSize = shard.getCompressedSize(); - UUID shardUuid = shard.getShardUuid(); - - // verify backup exists - if (!backupStore.get().shardExists(shardUuid)) { - log.warn("No backup for shard: %s", shardUuid); - } - - // pick target node - String target = pickTargetNode(nodes, shardSize, averageSize); - if (target == null) { - return; - } - long targetSize = nodes.get(target); - - // stats - log.info("Moving shard %s to node %s (shard: %s, node: %s, average: %s, target: %s)", - shardUuid, target, shardSize, nodeSize, averageSize, targetSize); - shardsEjected.update(1); - - // update size - nodes.put(target, targetSize + shardSize); - nodeSize -= shardSize; - - // move assignment - shardManager.replaceShardAssignment(shard.getTableId(), shardUuid, target, false); - - // delete local file - File file = storageService.getStorageFile(shardUuid); - if (file.exists() && !file.delete()) { - log.warn("Failed to delete shard file: %s", file); - } - } - } - - private static String pickTargetNode(Map nodes, long shardSize, long maxSize) - { - while (!nodes.isEmpty()) { - String node = pickCandidateNode(nodes); - if ((nodes.get(node) + shardSize) <= maxSize) { - return node; - } - nodes.remove(node); - } - return null; - } - - private static String pickCandidateNode(Map nodes) - { - checkArgument(!nodes.isEmpty()); - if (nodes.size() == 1) { - return nodes.keySet().iterator().next(); - } - - // pick two random candidates, then choose the smaller one - List candidates = new ArrayList<>(nodes.keySet()); - Collections.shuffle(candidates); - String first = candidates.get(0); - String second = candidates.get(1); - return (nodes.get(first) <= nodes.get(second)) ? first : second; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java deleted file mode 100644 index 2a416e31eecc..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.util.PrioritizedFifoExecutor; -import io.trino.spi.NodeManager; -import io.trino.spi.TrinoException; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.gaul.modernizer_maven_annotations.SuppressModernizer; -import org.weakref.jmx.Flatten; -import org.weakref.jmx.Managed; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.util.Comparator; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination; -import static io.airlift.concurrent.MoreFutures.addExceptionCallback; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.units.DataSize.succinctBytes; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_ERROR; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; -import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toSet; - -public class ShardRecoveryManager -{ - private static final Logger log = Logger.get(ShardRecoveryManager.class); - - private final StorageService storageService; - private final Optional backupStore; - private final String nodeIdentifier; - private final ShardManager shardManager; - private final Duration missingShardDiscoveryInterval; - - private final AtomicBoolean started = new AtomicBoolean(); - private final MissingShardsQueue shardQueue; - - private final ScheduledExecutorService missingShardExecutor = newScheduledThreadPool(1, daemonThreadsNamed("missing-shard-discovery")); - private final ExecutorService executorService = newCachedThreadPool(daemonThreadsNamed("shard-recovery-%s")); - private final ShardRecoveryStats stats; - - @Inject - public ShardRecoveryManager( - StorageService storageService, - Optional backupStore, - NodeManager nodeManager, - ShardManager shardManager, - StorageManagerConfig config) - { - this(storageService, - backupStore, - nodeManager, - shardManager, - config.getMissingShardDiscoveryInterval(), - config.getRecoveryThreads()); - } - - public ShardRecoveryManager( - StorageService storageService, - Optional backupStore, - NodeManager nodeManager, - ShardManager shardManager, - Duration missingShardDiscoveryInterval, - int recoveryThreads) - { - this.storageService = requireNonNull(storageService, "storageService is null"); - this.backupStore = requireNonNull(backupStore, "backupStore is null"); - this.nodeIdentifier = nodeManager.getCurrentNode().getNodeIdentifier(); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.missingShardDiscoveryInterval = requireNonNull(missingShardDiscoveryInterval, "missingShardDiscoveryInterval is null"); - this.shardQueue = new MissingShardsQueue(new PrioritizedFifoExecutor<>(executorService, recoveryThreads, new MissingShardComparator())); - this.stats = new ShardRecoveryStats(); - } - - @PostConstruct - public void start() - { - if (backupStore.isEmpty()) { - return; - } - if (started.compareAndSet(false, true)) { - scheduleRecoverMissingShards(); - } - } - - @PreDestroy - public void shutdown() - { - shutdownAndAwaitTermination(missingShardExecutor, 10, SECONDS); - shutdownAndAwaitTermination(executorService, 10, SECONDS); - } - - private void scheduleRecoverMissingShards() - { - missingShardExecutor.scheduleWithFixedDelay(() -> { - try { - // jitter to avoid overloading database - long interval = missingShardDiscoveryInterval.roundTo(SECONDS); - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval)); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - enqueueMissingShards(); - }, 0, missingShardDiscoveryInterval.toMillis(), TimeUnit.MILLISECONDS); - } - - @Managed - public void recoverMissingShards() - { - missingShardExecutor.submit(this::enqueueMissingShards); - } - - private synchronized void enqueueMissingShards() - { - try { - for (ShardMetadata shard : getMissingShards()) { - stats.incrementBackgroundShardRecovery(); - ListenableFuture future = shardQueue.submit(new MissingShard(shard.getShardUuid(), shard.getCompressedSize(), shard.getXxhash64(), false)); - addExceptionCallback(future, t -> log.warn(t, "Error recovering shard: %s", shard.getShardUuid())); - } - } - catch (Throwable t) { - if (!executorService.isShutdown()) { - log.error(t, "Error creating shard recovery tasks"); - } - } - } - - private Set getMissingShards() - { - return shardManager.getNodeShards(nodeIdentifier).stream() - .filter(shard -> shardNeedsRecovery(shard.getShardUuid(), shard.getCompressedSize())) - .collect(toSet()); - } - - private boolean shardNeedsRecovery(UUID shardUuid, long shardSize) - { - File storageFile = storageService.getStorageFile(shardUuid); - return !storageFile.exists() || (storageFile.length() != shardSize); - } - - public Future recoverShard(UUID shardUuid) - throws ExecutionException - { - ShardMetadata shard = shardManager.getShard(shardUuid); - if (shard == null) { - throw new TrinoException(RAPTOR_ERROR, "Shard does not exist in database: " + shardUuid); - } - stats.incrementActiveShardRecovery(); - return shardQueue.submit(new MissingShard(shardUuid, shard.getCompressedSize(), shard.getXxhash64(), true)); - } - - @VisibleForTesting - void restoreFromBackup(UUID shardUuid, long shardSize, OptionalLong shardXxhash64) - { - File storageFile = storageService.getStorageFile(shardUuid); - - if (!backupStore.get().shardExists(shardUuid)) { - stats.incrementShardRecoveryBackupNotFound(); - throw new TrinoException(RAPTOR_RECOVERY_ERROR, "No backup file found for shard: " + shardUuid); - } - - if (storageFile.exists()) { - if (!isFileCorrupt(storageFile, shardSize, shardXxhash64)) { - return; - } - stats.incrementCorruptLocalFile(); - quarantineFile(shardUuid, storageFile, "Local file is corrupt."); - } - - // create a temporary file in the staging directory - File stagingFile = temporarySuffix(storageService.getStagingFile(shardUuid)); - storageService.createParents(stagingFile); - - // copy to temporary file - log.info("Copying shard %s from backup...", shardUuid); - long start = System.nanoTime(); - - try { - backupStore.get().restoreShard(shardUuid, stagingFile); - } - catch (TrinoException e) { - stats.incrementShardRecoveryFailure(); - stagingFile.delete(); - throw e; - } - - Duration duration = nanosSince(start); - DataSize size = succinctBytes(stagingFile.length()); - DataSize rate = dataRate(size, duration).succinct(); - stats.addShardRecoveryDataRate(rate, size, duration); - - log.info("Copied shard %s from backup in %s (%s at %s/s)", shardUuid, duration, size, rate); - - // move to final location - storageService.createParents(storageFile); - try { - Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE); - } - catch (FileAlreadyExistsException e) { - // someone else already created it (should not happen, but safe to ignore) - } - catch (IOException e) { - stats.incrementShardRecoveryFailure(); - throw new TrinoException(RAPTOR_RECOVERY_ERROR, "Failed to move shard: " + shardUuid, e); - } - finally { - stagingFile.delete(); - } - - if (!storageFile.exists()) { - stats.incrementShardRecoveryFailure(); - throw new TrinoException(RAPTOR_RECOVERY_ERROR, "File does not exist after recovery: " + shardUuid); - } - - if (isFileCorrupt(storageFile, shardSize, shardXxhash64)) { - stats.incrementShardRecoveryFailure(); - stats.incrementCorruptRecoveredFile(); - quarantineFile(shardUuid, storageFile, "Local file is corrupt after recovery."); - throw new TrinoException(RAPTOR_BACKUP_CORRUPTION, "Backup is corrupt after read: " + shardUuid); - } - - stats.incrementShardRecoverySuccess(); - } - - private void quarantineFile(UUID shardUuid, File file, String message) - { - File quarantine = new File(storageService.getQuarantineFile(shardUuid).getPath() + ".corrupt"); - if (quarantine.exists()) { - log.warn("%s Quarantine already exists: %s", message, quarantine); - return; - } - - log.error("%s Quarantining corrupt file: %s", message, quarantine); - try { - Files.move(file.toPath(), quarantine.toPath(), ATOMIC_MOVE); - } - catch (IOException e) { - log.warn(e, "Quarantine of corrupt file failed: %s", quarantine); - file.delete(); - } - } - - private static boolean isFileCorrupt(File file, long size, OptionalLong xxhash64) - { - return (file.length() != size) || (xxhash64.isPresent() && (xxhash64(file) != xxhash64.getAsLong())); - } - - @VisibleForTesting - static class MissingShardComparator - implements Comparator - { - @Override - public int compare(MissingShardRunnable shard1, MissingShardRunnable shard2) - { - if (shard1.isActive() == shard2.isActive()) { - return 0; - } - return shard1.isActive() ? -1 : 1; - } - } - - interface MissingShardRunnable - extends Runnable - { - boolean isActive(); - } - - private class MissingShardRecovery - implements MissingShardRunnable - { - private final UUID shardUuid; - private final long shardSize; - private final OptionalLong shardXxhash64; - private final boolean active; - - public MissingShardRecovery(UUID shardUuid, long shardSize, OptionalLong shardXxhash64, boolean active) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.shardSize = shardSize; - this.shardXxhash64 = requireNonNull(shardXxhash64, "shardXxhash64 is null"); - this.active = active; - } - - @Override - public void run() - { - restoreFromBackup(shardUuid, shardSize, shardXxhash64); - } - - @Override - public boolean isActive() - { - return active; - } - } - - private static final class MissingShard - { - private final UUID shardUuid; - private final long shardSize; - private final OptionalLong shardXxhash64; - private final boolean active; - - public MissingShard(UUID shardUuid, long shardSize, OptionalLong shardXxhash64, boolean active) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.shardSize = shardSize; - this.shardXxhash64 = requireNonNull(shardXxhash64, "shardXxhash64 is null"); - this.active = active; - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public long getShardSize() - { - return shardSize; - } - - public OptionalLong getShardXxhash64() - { - return shardXxhash64; - } - - public boolean isActive() - { - return active; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - MissingShard other = (MissingShard) o; - return this.active == other.active && - Objects.equals(this.shardUuid, other.shardUuid); - } - - @Override - public int hashCode() - { - return Objects.hash(shardUuid, active); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuid", shardUuid) - .add("active", active) - .toString(); - } - } - - private class MissingShardsQueue - { - private final LoadingCache> queuedMissingShards; - - public MissingShardsQueue(PrioritizedFifoExecutor shardRecoveryExecutor) - { - requireNonNull(shardRecoveryExecutor, "shardRecoveryExecutor is null"); - this.queuedMissingShards = buildUnsafeCache(CacheBuilder.newBuilder(), new CacheLoader<>() - { - @Override - public ListenableFuture load(MissingShard missingShard) - { - MissingShardRecovery task = new MissingShardRecovery( - missingShard.getShardUuid(), - missingShard.getShardSize(), - missingShard.getShardXxhash64(), - missingShard.isActive()); - ListenableFuture future = shardRecoveryExecutor.submit(task); - // TODO (https://github.com/trinodb/trino/issues/10688) invalidation here races with `.load()` completion - future.addListener(() -> queuedMissingShards.invalidate(missingShard), directExecutor()); - return future; - } - }); - } - - // TODO (https://github.com/trinodb/trino/issues/10688) there is a load/invalidation race, so it's an unsafe suppression - @SuppressModernizer - private LoadingCache buildUnsafeCache(CacheBuilder cacheBuilder, CacheLoader cacheLoader) - { - return cacheBuilder.build(cacheLoader); - } - - public ListenableFuture submit(MissingShard shard) - throws ExecutionException - { - return queuedMissingShards.get(shard); - } - } - - static DataSize dataRate(DataSize size, Duration duration) - { - double rate = size.toBytes() / duration.getValue(SECONDS); - if (Double.isNaN(rate) || Double.isInfinite(rate)) { - rate = 0; - } - return succinctBytes(Math.round(rate)); - } - - private static File temporarySuffix(File file) - { - return new File(file.getPath() + ".tmp-" + UUID.randomUUID()); - } - - @Managed - @Flatten - public ShardRecoveryStats getStats() - { - return stats; - } - - private static FutureCallback failureCallback(Consumer callback) - { - return new FutureCallback<>() - { - @Override - public void onSuccess(T result) {} - - @Override - public void onFailure(Throwable throwable) - { - callback.accept(throwable); - } - }; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryStats.java deleted file mode 100644 index ff25a862cdb4..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryStats.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.errorprone.annotations.ThreadSafe; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -@ThreadSafe -public class ShardRecoveryStats -{ - private final CounterStat activeShardRecovery = new CounterStat(); - private final CounterStat backgroundShardRecovery = new CounterStat(); - private final CounterStat shardRecoverySuccess = new CounterStat(); - private final CounterStat shardRecoveryFailure = new CounterStat(); - private final CounterStat shardRecoveryBackupNotFound = new CounterStat(); - - private final DistributionStat shardRecoveryShardSizeBytes = new DistributionStat(); - private final DistributionStat shardRecoveryTimeInMilliSeconds = new DistributionStat(); - private final DistributionStat shardRecoveryBytesPerSecond = new DistributionStat(); - - private final CounterStat corruptLocalFile = new CounterStat(); - private final CounterStat corruptRecoveredFile = new CounterStat(); - - public void incrementBackgroundShardRecovery() - { - backgroundShardRecovery.update(1); - } - - public void incrementActiveShardRecovery() - { - activeShardRecovery.update(1); - } - - public void incrementShardRecoveryBackupNotFound() - { - shardRecoveryBackupNotFound.update(1); - } - - public void incrementShardRecoveryFailure() - { - shardRecoveryFailure.update(1); - } - - public void incrementShardRecoverySuccess() - { - shardRecoverySuccess.update(1); - } - - public void addShardRecoveryDataRate(DataSize rate, DataSize size, Duration duration) - { - shardRecoveryBytesPerSecond.add(rate.toBytes()); - shardRecoveryShardSizeBytes.add(size.toBytes()); - shardRecoveryTimeInMilliSeconds.add(duration.toMillis()); - } - - public void incrementCorruptLocalFile() - { - corruptLocalFile.update(1); - } - - public void incrementCorruptRecoveredFile() - { - corruptRecoveredFile.update(1); - } - - @Managed - @Nested - public CounterStat getActiveShardRecovery() - { - return activeShardRecovery; - } - - @Managed - @Nested - public CounterStat getBackgroundShardRecovery() - { - return backgroundShardRecovery; - } - - @Managed - @Nested - public CounterStat getShardRecoverySuccess() - { - return shardRecoverySuccess; - } - - @Managed - @Nested - public CounterStat getShardRecoveryFailure() - { - return shardRecoveryFailure; - } - - @Managed - @Nested - public CounterStat getShardRecoveryBackupNotFound() - { - return shardRecoveryBackupNotFound; - } - - @Managed - @Nested - public DistributionStat getShardRecoveryBytesPerSecond() - { - return shardRecoveryBytesPerSecond; - } - - @Managed - @Nested - public DistributionStat getShardRecoveryTimeInMilliSeconds() - { - return shardRecoveryTimeInMilliSeconds; - } - - @Managed - @Nested - public DistributionStat getShardRecoveryShardSizeBytes() - { - return shardRecoveryShardSizeBytes; - } - - @Managed - @Nested - public CounterStat getCorruptLocalFile() - { - return corruptLocalFile; - } - - @Managed - @Nested - public CounterStat getCorruptRecoveredFile() - { - return corruptRecoveredFile; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRewriter.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRewriter.java deleted file mode 100644 index 8d1645c78849..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRewriter.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.airlift.slice.Slice; - -import java.util.BitSet; -import java.util.Collection; -import java.util.concurrent.CompletableFuture; - -public interface ShardRewriter -{ - CompletableFuture> rewrite(BitSet rowsToDelete); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java deleted file mode 100644 index 4909d56e2b8c..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.orc.OrcColumn; -import io.trino.orc.OrcPredicate; -import io.trino.orc.OrcReader; -import io.trino.orc.OrcRecordReader; -import io.trino.plugin.raptor.legacy.metadata.ColumnStats; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.VarcharType; - -import java.io.IOException; -import java.util.List; -import java.util.Optional; - -import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static io.trino.orc.OrcReader.INITIAL_BATCH_SIZE; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.toOrcFileType; -import static java.lang.Double.isInfinite; -import static java.lang.Double.isNaN; -import static org.joda.time.DateTimeZone.UTC; - -public final class ShardStats -{ - /** - * Maximum length of a binary value stored in an index. - */ - public static final int MAX_BINARY_INDEX_SIZE = 100; - - private ShardStats() {} - - public static Slice truncateIndexValue(Slice slice) - { - if (slice.length() > MAX_BINARY_INDEX_SIZE) { - return slice.slice(0, MAX_BINARY_INDEX_SIZE); - } - return slice; - } - - public static Optional computeColumnStats(OrcReader orcReader, long columnId, Type type, TypeManager typeManager) - throws IOException - { - return Optional.ofNullable(doComputeColumnStats(orcReader, columnId, type, typeManager)); - } - - private static ColumnStats doComputeColumnStats(OrcReader orcReader, long columnId, Type type, TypeManager typeManager) - throws IOException - { - OrcColumn column = getColumn(orcReader.getRootColumn().getNestedColumns(), columnId); - Type columnType = toOrcFileType(type, typeManager); - OrcRecordReader reader = orcReader.createRecordReader( - ImmutableList.of(column), - ImmutableList.of(columnType), - OrcPredicate.TRUE, - UTC, - newSimpleAggregatedMemoryContext(), - INITIAL_BATCH_SIZE, - exception -> new TrinoException(RAPTOR_ERROR, "Error reading column: " + columnId, exception)); - - if (type.equals(BooleanType.BOOLEAN)) { - return indexBoolean(reader, columnId); - } - if (type.equals(BigintType.BIGINT) || - type.equals(DateType.DATE) || - type.equals(TimestampType.TIMESTAMP_MILLIS)) { - return indexLong(type, reader, columnId); - } - if (type.equals(DoubleType.DOUBLE)) { - return indexDouble(reader, columnId); - } - if (type instanceof VarcharType) { - return indexString(type, reader, columnId); - } - return null; - } - - private static OrcColumn getColumn(List columnNames, long columnId) - { - String columnName = String.valueOf(columnId); - return columnNames.stream() - .filter(column -> column.getColumnName().equals(columnName)) - .findFirst() - .orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "Missing column ID: " + columnId)); - } - - private static ColumnStats indexBoolean(OrcRecordReader reader, long columnId) - throws IOException - { - boolean minSet = false; - boolean maxSet = false; - boolean min = false; - boolean max = false; - - while (true) { - Page page = reader.nextPage(); - if (page == null) { - break; - } - Block block = page.getBlock(0).getLoadedBlock(); - - for (int i = 0; i < page.getPositionCount(); i++) { - if (block.isNull(i)) { - continue; - } - boolean value = BooleanType.BOOLEAN.getBoolean(block, i); - if (!minSet || Boolean.compare(value, min) < 0) { - minSet = true; - min = value; - } - if (!maxSet || Boolean.compare(value, max) > 0) { - maxSet = true; - max = value; - } - } - } - - return new ColumnStats(columnId, - minSet ? min : null, - maxSet ? max : null); - } - - private static ColumnStats indexLong(Type type, OrcRecordReader reader, long columnId) - throws IOException - { - boolean minSet = false; - boolean maxSet = false; - long min = 0; - long max = 0; - - while (true) { - Page page = reader.nextPage(); - if (page == null) { - break; - } - Block block = page.getBlock(0).getLoadedBlock(); - - for (int i = 0; i < page.getPositionCount(); i++) { - if (block.isNull(i)) { - continue; - } - long value = type.getLong(block, i); - if (!minSet || (value < min)) { - minSet = true; - min = value; - } - if (!maxSet || (value > max)) { - maxSet = true; - max = value; - } - } - } - - return new ColumnStats(columnId, - minSet ? min : null, - maxSet ? max : null); - } - - private static ColumnStats indexDouble(OrcRecordReader reader, long columnId) - throws IOException - { - boolean minSet = false; - boolean maxSet = false; - double min = 0; - double max = 0; - - while (true) { - Page page = reader.nextPage(); - if (page == null) { - break; - } - Block block = page.getBlock(0).getLoadedBlock(); - - for (int i = 0; i < page.getPositionCount(); i++) { - if (block.isNull(i)) { - continue; - } - double value = DoubleType.DOUBLE.getDouble(block, i); - if (isNaN(value)) { - continue; - } - if (value == -0.0) { - value = 0.0; - } - if (!minSet || (value < min)) { - minSet = true; - min = value; - } - if (!maxSet || (value > max)) { - maxSet = true; - max = value; - } - } - } - - if (isInfinite(min)) { - minSet = false; - } - if (isInfinite(max)) { - maxSet = false; - } - - return new ColumnStats(columnId, - minSet ? min : null, - maxSet ? max : null); - } - - private static ColumnStats indexString(Type type, OrcRecordReader reader, long columnId) - throws IOException - { - boolean minSet = false; - boolean maxSet = false; - Slice min = null; - Slice max = null; - - while (true) { - Page page = reader.nextPage(); - if (page == null) { - break; - } - Block block = page.getBlock(0).getLoadedBlock(); - - for (int i = 0; i < page.getPositionCount(); i++) { - if (block.isNull(i)) { - continue; - } - Slice slice = type.getSlice(block, i); - slice = truncateIndexValue(slice); - if (!minSet || (slice.compareTo(min) < 0)) { - minSet = true; - min = slice; - } - if (!maxSet || (slice.compareTo(max) > 0)) { - maxSet = true; - max = slice; - } - } - } - - return new ColumnStats(columnId, - minSet ? min.toStringUtf8() : null, - maxSet ? max.toStringUtf8() : null); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManager.java deleted file mode 100644 index 628ef717a4e2..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManager.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.trino.orc.OrcReaderOptions; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.OptionalInt; -import java.util.UUID; - -public interface StorageManager -{ - ConnectorPageSource getPageSource( - UUID shardUuid, - OptionalInt bucketNumber, - List columnIds, - List columnTypes, - TupleDomain effectivePredicate, - OrcReaderOptions orcReaderOptions); - - StoragePageSink createStoragePageSink( - long transactionId, - OptionalInt bucketNumber, - List columnIds, - List columnTypes, - boolean checkSpace); - - ShardRewriter createShardRewriter( - long transactionId, - OptionalInt bucketNumber, - UUID shardUuid); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManagerConfig.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManagerConfig.java deleted file mode 100644 index 2ce5a87410cb..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageManagerConfig.java +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; -import io.airlift.configuration.LegacyConfig; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.airlift.units.MaxDataSize; -import io.airlift.units.MinDataSize; -import io.airlift.units.MinDuration; -import io.trino.orc.OrcReaderOptions; -import jakarta.validation.constraints.Max; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; - -import java.io.File; -import java.util.concurrent.TimeUnit; - -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static java.lang.Math.max; -import static java.lang.Runtime.getRuntime; - -@DefunctConfig({ - "storage.backup-directory", - "storage.shard-day-boundary-time-zone", -}) -public class StorageManagerConfig -{ - private File dataDirectory; - private DataSize minAvailableSpace = DataSize.ofBytes(0); - private Duration shardRecoveryTimeout = new Duration(30, TimeUnit.SECONDS); - private Duration missingShardDiscoveryInterval = new Duration(5, TimeUnit.MINUTES); - private boolean compactionEnabled = true; - private Duration compactionInterval = new Duration(1, TimeUnit.HOURS); - private Duration shardEjectorInterval = new Duration(4, TimeUnit.HOURS); - private OrcReaderOptions options = new OrcReaderOptions(); - private int deletionThreads = max(1, getRuntime().availableProcessors() / 2); - private int recoveryThreads = 10; - private int organizationThreads = 5; - private boolean organizationEnabled = true; - private Duration organizationDiscoveryInterval = new Duration(6, TimeUnit.HOURS); - private Duration organizationInterval = new Duration(7, TimeUnit.DAYS); - - private long maxShardRows = 1_000_000; - private DataSize maxShardSize = DataSize.of(256, MEGABYTE); - private DataSize maxBufferSize = DataSize.of(256, MEGABYTE); - private int oneSplitPerBucketThreshold; - - @NotNull - public File getDataDirectory() - { - return dataDirectory; - } - - @Config("storage.data-directory") - @ConfigDescription("Base directory to use for storing shard data") - public StorageManagerConfig setDataDirectory(File dataDirectory) - { - this.dataDirectory = dataDirectory; - return this; - } - - @NotNull - public DataSize getMinAvailableSpace() - { - return minAvailableSpace; - } - - @Config("storage.min-available-space") - @ConfigDescription("Minimum space that must be available on the data directory file system") - public StorageManagerConfig setMinAvailableSpace(DataSize minAvailableSpace) - { - this.minAvailableSpace = minAvailableSpace; - return this; - } - - public OrcReaderOptions toOrcReaderOptions() - { - return options; - } - - @NotNull - public DataSize getOrcMaxMergeDistance() - { - return options.getMaxMergeDistance(); - } - - @Config("storage.orc.max-merge-distance") - public StorageManagerConfig setOrcMaxMergeDistance(DataSize orcMaxMergeDistance) - { - options = options.withMaxMergeDistance(orcMaxMergeDistance); - return this; - } - - @NotNull - public DataSize getOrcMaxReadSize() - { - return options.getMaxBufferSize(); - } - - @Config("storage.orc.max-read-size") - public StorageManagerConfig setOrcMaxReadSize(DataSize orcMaxReadSize) - { - options = options.withMaxBufferSize(orcMaxReadSize); - return this; - } - - @NotNull - public DataSize getOrcStreamBufferSize() - { - return options.getStreamBufferSize(); - } - - @Config("storage.orc.stream-buffer-size") - public StorageManagerConfig setOrcStreamBufferSize(DataSize orcStreamBufferSize) - { - options = options.withStreamBufferSize(orcStreamBufferSize); - return this; - } - - @NotNull - public DataSize getOrcTinyStripeThreshold() - { - return options.getTinyStripeThreshold(); - } - - @Config("storage.orc.tiny-stripe-threshold") - public StorageManagerConfig setOrcTinyStripeThreshold(DataSize orcTinyStripeThreshold) - { - options = options.withTinyStripeThreshold(orcTinyStripeThreshold); - return this; - } - - @Deprecated - public boolean isOrcLazyReadSmallRanges() - { - return options.isLazyReadSmallRanges(); - } - - // TODO remove config option once efficacy is proven - @Deprecated - @Config("storage.orc.lazy-read-small-ranges") - public StorageManagerConfig setOrcLazyReadSmallRanges(boolean orcLazyReadSmallRanges) - { - options = options.withLazyReadSmallRanges(orcLazyReadSmallRanges); - return this; - } - - @Deprecated - public boolean isOrcNestedLazy() - { - return options.isNestedLazy(); - } - - // TODO remove config option once efficacy is proven - @Deprecated - @Config("storage.orc.nested-lazy") - public StorageManagerConfig setOrcNestedLazy(boolean nestedLazy) - { - options = options.withNestedLazy(nestedLazy); - return this; - } - - @Min(1) - public int getDeletionThreads() - { - return deletionThreads; - } - - @Config("storage.max-deletion-threads") - @ConfigDescription("Maximum number of threads to use for deletions") - public StorageManagerConfig setDeletionThreads(int deletionThreads) - { - this.deletionThreads = deletionThreads; - return this; - } - - @MinDuration("1s") - public Duration getShardRecoveryTimeout() - { - return shardRecoveryTimeout; - } - - @Config("storage.shard-recovery-timeout") - @ConfigDescription("Maximum time to wait for a shard to recover from backup while running a query") - public StorageManagerConfig setShardRecoveryTimeout(Duration shardRecoveryTimeout) - { - this.shardRecoveryTimeout = shardRecoveryTimeout; - return this; - } - - @MinDuration("1s") - public Duration getMissingShardDiscoveryInterval() - { - return missingShardDiscoveryInterval; - } - - @Config("storage.missing-shard-discovery-interval") - @ConfigDescription("How often to check the database and local file system missing shards") - public StorageManagerConfig setMissingShardDiscoveryInterval(Duration missingShardDiscoveryInterval) - { - this.missingShardDiscoveryInterval = missingShardDiscoveryInterval; - return this; - } - - @MinDuration("1s") - public Duration getCompactionInterval() - { - return compactionInterval; - } - - @Config("storage.compaction-interval") - @ConfigDescription("How often to check for local shards that need compaction") - public StorageManagerConfig setCompactionInterval(Duration compactionInterval) - { - this.compactionInterval = compactionInterval; - return this; - } - - @NotNull - @MinDuration("1s") - public Duration getOrganizationInterval() - { - return organizationInterval; - } - - @Config("storage.organization-interval") - @ConfigDescription("How long to wait between table organization iterations") - public StorageManagerConfig setOrganizationInterval(Duration organizationInterval) - { - this.organizationInterval = organizationInterval; - return this; - } - - @NotNull - @MinDuration("1s") - public Duration getOrganizationDiscoveryInterval() - { - return organizationDiscoveryInterval; - } - - @Config("storage.organization-discovery-interval") - @ConfigDescription("How long to wait between discovering tables that need to be organized") - public StorageManagerConfig setOrganizationDiscoveryInterval(Duration organizationDiscoveryInterval) - { - this.organizationDiscoveryInterval = organizationDiscoveryInterval; - return this; - } - - @MinDuration("5m") - public Duration getShardEjectorInterval() - { - return shardEjectorInterval; - } - - @Config("storage.ejector-interval") - @ConfigDescription("How often to check for local shards that need ejection to balance capacity") - public StorageManagerConfig setShardEjectorInterval(Duration shardEjectorInterval) - { - this.shardEjectorInterval = shardEjectorInterval; - return this; - } - - @Min(1) - public int getRecoveryThreads() - { - return recoveryThreads; - } - - @Config("storage.max-recovery-threads") - @ConfigDescription("Maximum number of threads to use for recovery") - public StorageManagerConfig setRecoveryThreads(int recoveryThreads) - { - this.recoveryThreads = recoveryThreads; - return this; - } - - @LegacyConfig("storage.max-compaction-threads") - @Config("storage.max-organization-threads") - @ConfigDescription("Maximum number of threads to use for organization") - public StorageManagerConfig setOrganizationThreads(int organizationThreads) - { - this.organizationThreads = organizationThreads; - return this; - } - - @Min(1) - public int getOrganizationThreads() - { - return organizationThreads; - } - - @Min(1) - @Max(1_000_000_000) - public long getMaxShardRows() - { - return maxShardRows; - } - - @Config("storage.max-shard-rows") - @ConfigDescription("Approximate maximum number of rows per shard") - public StorageManagerConfig setMaxShardRows(long maxShardRows) - { - this.maxShardRows = maxShardRows; - return this; - } - - @MinDataSize("1MB") - @MaxDataSize("1GB") - public DataSize getMaxShardSize() - { - return maxShardSize; - } - - @Config("storage.max-shard-size") - @ConfigDescription("Approximate maximum uncompressed size of a shard") - public StorageManagerConfig setMaxShardSize(DataSize maxShardSize) - { - this.maxShardSize = maxShardSize; - return this; - } - - @MinDataSize("1MB") - public DataSize getMaxBufferSize() - { - return maxBufferSize; - } - - @Config("storage.max-buffer-size") - @ConfigDescription("Maximum data to buffer before flushing to disk") - public StorageManagerConfig setMaxBufferSize(DataSize maxBufferSize) - { - this.maxBufferSize = maxBufferSize; - return this; - } - - public boolean isCompactionEnabled() - { - return compactionEnabled; - } - - @Config("storage.compaction-enabled") - public StorageManagerConfig setCompactionEnabled(boolean compactionEnabled) - { - this.compactionEnabled = compactionEnabled; - return this; - } - - public boolean isOrganizationEnabled() - { - return organizationEnabled; - } - - @Config("storage.organization-enabled") - public StorageManagerConfig setOrganizationEnabled(boolean organizationEnabled) - { - this.organizationEnabled = organizationEnabled; - return this; - } - - public int getOneSplitPerBucketThreshold() - { - return oneSplitPerBucketThreshold; - } - - @Config("storage.one-split-per-bucket-threshold") - @ConfigDescription("Experimental: Maximum bucket count at which to produce multiple splits per bucket") - public StorageManagerConfig setOneSplitPerBucketThreshold(int oneSplitPerBucketThreshold) - { - this.oneSplitPerBucketThreshold = oneSplitPerBucketThreshold; - return this; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java deleted file mode 100644 index 5c0d0c13ccbd..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.base.Ticker; -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Scopes; -import io.trino.plugin.raptor.legacy.backup.BackupManager; -import io.trino.plugin.raptor.legacy.metadata.AssignmentLimiter; -import io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager; -import io.trino.plugin.raptor.legacy.metadata.DatabaseShardRecorder; -import io.trino.plugin.raptor.legacy.metadata.MetadataConfig; -import io.trino.plugin.raptor.legacy.metadata.ShardCleaner; -import io.trino.plugin.raptor.legacy.metadata.ShardCleanerConfig; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; -import io.trino.plugin.raptor.legacy.storage.organization.JobFactory; -import io.trino.plugin.raptor.legacy.storage.organization.OrganizationJobFactory; -import io.trino.plugin.raptor.legacy.storage.organization.ShardCompactionManager; -import io.trino.plugin.raptor.legacy.storage.organization.ShardCompactor; -import io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizationManager; -import io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizer; - -import static io.airlift.configuration.ConfigBinder.configBinder; -import static org.weakref.jmx.guice.ExportBinder.newExporter; - -public class StorageModule - implements Module -{ - @Override - public void configure(Binder binder) - { - configBinder(binder).bindConfig(StorageManagerConfig.class); - configBinder(binder).bindConfig(BucketBalancerConfig.class); - configBinder(binder).bindConfig(ShardCleanerConfig.class); - configBinder(binder).bindConfig(MetadataConfig.class); - - binder.bind(Ticker.class).toInstance(Ticker.systemTicker()); - - binder.bind(StorageManager.class).to(RaptorStorageManager.class).in(Scopes.SINGLETON); - binder.bind(StorageService.class).to(FileStorageService.class).in(Scopes.SINGLETON); - binder.bind(ShardManager.class).to(DatabaseShardManager.class).in(Scopes.SINGLETON); - binder.bind(ShardRecorder.class).to(DatabaseShardRecorder.class).in(Scopes.SINGLETON); - binder.bind(DatabaseShardManager.class).in(Scopes.SINGLETON); - binder.bind(DatabaseShardRecorder.class).in(Scopes.SINGLETON); - binder.bind(ShardRecoveryManager.class).in(Scopes.SINGLETON); - binder.bind(BackupManager.class).in(Scopes.SINGLETON); - binder.bind(ShardCompactionManager.class).in(Scopes.SINGLETON); - binder.bind(ShardOrganizationManager.class).in(Scopes.SINGLETON); - binder.bind(ShardOrganizer.class).in(Scopes.SINGLETON); - binder.bind(JobFactory.class).to(OrganizationJobFactory.class).in(Scopes.SINGLETON); - binder.bind(ShardCompactor.class).in(Scopes.SINGLETON); - binder.bind(ShardEjector.class).in(Scopes.SINGLETON); - binder.bind(ShardCleaner.class).in(Scopes.SINGLETON); - binder.bind(BucketBalancer.class).in(Scopes.SINGLETON); - binder.bind(AssignmentLimiter.class).in(Scopes.SINGLETON); - - newExporter(binder).export(ShardRecoveryManager.class).withGeneratedName(); - newExporter(binder).export(BackupManager.class).withGeneratedName(); - newExporter(binder).export(StorageManager.class).as(generator -> generator.generatedNameOf(RaptorStorageManager.class)); - newExporter(binder).export(ShardCompactionManager.class).withGeneratedName(); - newExporter(binder).export(ShardOrganizer.class).withGeneratedName(); - newExporter(binder).export(ShardCompactor.class).withGeneratedName(); - newExporter(binder).export(ShardEjector.class).withGeneratedName(); - newExporter(binder).export(ShardCleaner.class).withGeneratedName(); - newExporter(binder).export(BucketBalancer.class).withGeneratedName(); - newExporter(binder).export(JobFactory.class).withGeneratedName(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StoragePageSink.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StoragePageSink.java deleted file mode 100644 index f999594f93ad..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StoragePageSink.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.spi.Page; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -public interface StoragePageSink -{ - default void appendPage(Page page) - { - appendPages(ImmutableList.of(page)); - } - - void appendPages(List pages); - - void appendPages(List pages, int[] pageIndexes, int[] positionIndexes); - - boolean isFull(); - - void flush(); - - CompletableFuture> commit(); - - void rollback(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageService.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageService.java deleted file mode 100644 index 05441257d529..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import java.io.File; -import java.util.Set; -import java.util.UUID; - -public interface StorageService -{ - void start(); - - long getAvailableBytes(); - - void createParents(File file); - - File getStorageFile(UUID shardUuid); - - File getStagingFile(UUID shardUuid); - - File getQuarantineFile(UUID shardUuid); - - Set getStorageShards(); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/CompactionSetCreator.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/CompactionSetCreator.java deleted file mode 100644 index 728063b2adbf..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/CompactionSetCreator.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableSet; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.metadata.Table; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.createOrganizationSet; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getShardsByDaysBuckets; -import static java.util.Comparator.comparing; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toCollection; - -public class CompactionSetCreator -{ - private final DataSize maxShardSize; - private final long maxShardRows; - - public CompactionSetCreator(DataSize maxShardSize, long maxShardRows) - { - checkArgument(maxShardRows > 0, "maxShardRows must be > 0"); - - this.maxShardSize = requireNonNull(maxShardSize, "maxShardSize is null"); - this.maxShardRows = maxShardRows; - } - - // Expects a pre-filtered collection of shards. - // All shards provided to this method will be considered for creating a compaction set. - public Set createCompactionSets(Table tableInfo, Collection shards) - { - Collection> shardsByDaysBuckets = getShardsByDaysBuckets(tableInfo, shards); - - ImmutableSet.Builder compactionSets = ImmutableSet.builder(); - for (Collection shardInfos : shardsByDaysBuckets) { - compactionSets.addAll(buildCompactionSets(tableInfo, ImmutableSet.copyOf(shardInfos))); - } - return compactionSets.build(); - } - - private Set buildCompactionSets(Table tableInfo, Set shardIndexInfos) - { - long tableId = tableInfo.getTableId(); - List shards = shardIndexInfos.stream() - .sorted(getShardIndexInfoComparator(tableInfo)) - .collect(toCollection(ArrayList::new)); - - long consumedBytes = 0; - long consumedRows = 0; - ImmutableSet.Builder builder = ImmutableSet.builder(); - ImmutableSet.Builder compactionSets = ImmutableSet.builder(); - - for (ShardIndexInfo shard : shards) { - if (((consumedBytes + shard.getUncompressedSize()) > maxShardSize.toBytes()) || - (consumedRows + shard.getRowCount() > maxShardRows)) { - // Finalize this compaction set, and start a new one for the rest of the shards - Set shardsToCompact = builder.build(); - - if (shardsToCompact.size() > 1) { - compactionSets.add(createOrganizationSet(tableId, shardsToCompact)); - } - - builder = ImmutableSet.builder(); - consumedBytes = 0; - consumedRows = 0; - } - builder.add(shard); - consumedBytes += shard.getUncompressedSize(); - consumedRows += shard.getRowCount(); - } - - // create compaction set for the remaining shards of this day - Set shardsToCompact = builder.build(); - if (shardsToCompact.size() > 1) { - compactionSets.add(createOrganizationSet(tableId, shardsToCompact)); - } - return compactionSets.build(); - } - - private static Comparator getShardIndexInfoComparator(Table tableInfo) - { - if (tableInfo.getTemporalColumnId().isEmpty()) { - return comparing(ShardIndexInfo::getUncompressedSize); - } - - return comparing(info -> info.getTemporalRange().get(), - comparing(ShardRange::getMinTuple) - .thenComparing(ShardRange::getMaxTuple)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/JobFactory.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/JobFactory.java deleted file mode 100644 index 1e979e8ca1c3..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/JobFactory.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -public interface JobFactory -{ - Runnable create(OrganizationSet organizationSet); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJob.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJob.java deleted file mode 100644 index 6a73bd9ec446..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJob.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import io.airlift.log.Logger; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.plugin.raptor.legacy.metadata.TableMetadata; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.List; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST; -import static java.util.Collections.nCopies; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -class OrganizationJob - implements Runnable -{ - private static final Logger log = Logger.get(OrganizationJob.class); - - private final MetadataDao metadataDao; - private final ShardManager shardManager; - private final ShardCompactor compactor; - private final OrganizationSet organizationSet; - - public OrganizationJob(OrganizationSet organizationSet, MetadataDao metadataDao, ShardManager shardManager, ShardCompactor compactor) - { - this.metadataDao = requireNonNull(metadataDao, "metadataDao is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.compactor = requireNonNull(compactor, "compactor is null"); - this.organizationSet = requireNonNull(organizationSet, "organizationSet is null"); - } - - @Override - public void run() - { - try { - runJob(organizationSet.getTableId(), organizationSet.getBucketNumber(), organizationSet.getShards()); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private void runJob(long tableId, OptionalInt bucketNumber, Set shardUuids) - throws IOException - { - long transactionId = shardManager.beginTransaction(); - try { - runJob(transactionId, bucketNumber, tableId, shardUuids); - } - catch (Throwable e) { - shardManager.rollbackTransaction(transactionId); - throw e; - } - } - - private void runJob(long transactionId, OptionalInt bucketNumber, long tableId, Set shardUuids) - throws IOException - { - TableMetadata metadata = getTableMetadata(tableId); - List newShards = performCompaction(transactionId, bucketNumber, shardUuids, metadata); - log.info("Compacted shards %s into %s", shardUuids, newShards.stream().map(ShardInfo::getShardUuid).collect(toList())); - shardManager.replaceShardUuids(transactionId, tableId, metadata.getColumns(), shardUuids, newShards, OptionalLong.empty()); - } - - private TableMetadata getTableMetadata(long tableId) - { - List sortColumns = metadataDao.listSortColumns(tableId); - - List sortColumnIds = sortColumns.stream() - .map(TableColumn::getColumnId) - .collect(toList()); - - List columns = metadataDao.listTableColumns(tableId).stream() - .map(TableColumn::toColumnInfo) - .collect(toList()); - return new TableMetadata(tableId, columns, sortColumnIds); - } - - private List performCompaction(long transactionId, OptionalInt bucketNumber, Set shardUuids, TableMetadata tableMetadata) - throws IOException - { - if (tableMetadata.getSortColumnIds().isEmpty()) { - return compactor.compact(transactionId, bucketNumber, shardUuids, tableMetadata.getColumns()); - } - return compactor.compactSorted( - transactionId, - bucketNumber, - shardUuids, - tableMetadata.getColumns(), - tableMetadata.getSortColumnIds(), - nCopies(tableMetadata.getSortColumnIds().size(), ASC_NULLS_FIRST)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJobFactory.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJobFactory.java deleted file mode 100644 index fc9e69134186..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationJobFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import org.jdbi.v3.core.Jdbi; - -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static java.util.Objects.requireNonNull; - -public class OrganizationJobFactory - implements JobFactory -{ - private final MetadataDao metadataDao; - private final ShardManager shardManager; - private final ShardCompactor compactor; - - @Inject - public OrganizationJobFactory(@ForMetadata Jdbi dbi, ShardManager shardManager, ShardCompactor compactor) - { - requireNonNull(dbi, "dbi is null"); - this.metadataDao = onDemandDao(dbi, MetadataDao.class); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.compactor = requireNonNull(compactor, "compactor is null"); - } - - @Override - public Runnable create(OrganizationSet organizationSet) - { - return new OrganizationJob(organizationSet, metadataDao, shardManager, compactor); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationSet.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationSet.java deleted file mode 100644 index 923e6f299ec5..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/OrganizationSet.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import java.util.Objects; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class OrganizationSet -{ - private final long tableId; - private final Set shards; - private final OptionalInt bucketNumber; - - public OrganizationSet(long tableId, Set shards, OptionalInt bucketNumber) - { - this.tableId = tableId; - this.shards = requireNonNull(shards, "shards is null"); - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - } - - public long getTableId() - { - return tableId; - } - - public Set getShards() - { - return shards; - } - - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OrganizationSet that = (OrganizationSet) o; - return tableId == that.tableId && - Objects.equals(shards, that.shards) && - Objects.equals(bucketNumber, that.bucketNumber); - } - - @Override - public int hashCode() - { - return Objects.hash(tableId, shards, bucketNumber); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("tableId", tableId) - .add("shards", shards) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactionManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactionManager.java deleted file mode 100644 index a8b554259b31..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactionManager.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Multimaps; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.NodeManager; -import io.trino.spi.type.Type; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.jdbi.v3.core.Jdbi; - -import java.util.Collection; -import java.util.List; -import java.util.Map.Entry; -import java.util.OptionalLong; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getOrganizationEligibleShards; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toSet; - -public class ShardCompactionManager -{ - private static final Logger log = Logger.get(ShardCompactionManager.class); - - private static final double FILL_FACTOR = 0.75; - - private final ScheduledExecutorService compactionDiscoveryService = newScheduledThreadPool(1, daemonThreadsNamed("shard-compaction-discovery")); - - private final AtomicBoolean discoveryStarted = new AtomicBoolean(); - - private final MetadataDao metadataDao; - private final ShardOrganizer organizer; - private final ShardManager shardManager; - private final String currentNodeIdentifier; - private final CompactionSetCreator compactionSetCreator; - - private final boolean compactionEnabled; - private final Duration compactionDiscoveryInterval; - private final DataSize maxShardSize; - private final long maxShardRows; - private final Jdbi dbi; - - @Inject - public ShardCompactionManager(@ForMetadata Jdbi dbi, - NodeManager nodeManager, - ShardManager shardManager, - ShardOrganizer organizer, - StorageManagerConfig config) - { - this(dbi, - nodeManager.getCurrentNode().getNodeIdentifier(), - shardManager, - organizer, - config.getCompactionInterval(), - config.getMaxShardSize(), - config.getMaxShardRows(), - config.isCompactionEnabled()); - } - - public ShardCompactionManager( - Jdbi dbi, - String currentNodeIdentifier, - ShardManager shardManager, - ShardOrganizer organizer, - Duration compactionDiscoveryInterval, - DataSize maxShardSize, - long maxShardRows, - boolean compactionEnabled) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.metadataDao = onDemandDao(dbi, MetadataDao.class); - - this.currentNodeIdentifier = requireNonNull(currentNodeIdentifier, "currentNodeIdentifier is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.organizer = requireNonNull(organizer, "organizer is null"); - this.compactionDiscoveryInterval = requireNonNull(compactionDiscoveryInterval, "compactionDiscoveryInterval is null"); - - checkArgument(maxShardSize.toBytes() > 0, "maxShardSize must be > 0"); - this.maxShardSize = requireNonNull(maxShardSize, "maxShardSize is null"); - - checkArgument(maxShardRows > 0, "maxShardRows must be > 0"); - this.maxShardRows = maxShardRows; - - this.compactionEnabled = compactionEnabled; - this.compactionSetCreator = new CompactionSetCreator(maxShardSize, maxShardRows); - } - - @PostConstruct - public void start() - { - if (!compactionEnabled) { - return; - } - - if (!discoveryStarted.getAndSet(true)) { - startDiscovery(); - } - } - - @PreDestroy - public void shutdown() - { - if (!compactionEnabled) { - return; - } - compactionDiscoveryService.shutdown(); - } - - private void startDiscovery() - { - compactionDiscoveryService.scheduleWithFixedDelay(() -> { - try { - // jitter to avoid overloading database - long interval = (long) compactionDiscoveryInterval.convertTo(SECONDS).getValue(); - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval)); - discoverShards(); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable t) { - log.error(t, "Error discovering shards to compact"); - } - }, 0, compactionDiscoveryInterval.toMillis(), TimeUnit.MILLISECONDS); - } - - private void discoverShards() - { - log.info("Discovering shards that need compaction..."); - Set allShards = shardManager.getNodeShards(currentNodeIdentifier); - ListMultimap tableShards = Multimaps.index(allShards, ShardMetadata::getTableId); - - for (Entry> entry : Multimaps.asMap(tableShards).entrySet()) { - long tableId = entry.getKey(); - if (!metadataDao.isCompactionEligible(tableId)) { - continue; - } - List shards = entry.getValue(); - Collection organizationSets = filterAndCreateCompactionSets(tableId, shards); - log.info("Created %s organization set(s) for table ID %s", organizationSets.size(), tableId); - - for (OrganizationSet set : organizationSets) { - organizer.enqueue(set); - } - } - } - - private Collection filterAndCreateCompactionSets(long tableId, Collection tableShards) - { - Table tableInfo = metadataDao.getTableInformation(tableId); - OptionalLong temporalColumnId = tableInfo.getTemporalColumnId(); - if (temporalColumnId.isPresent()) { - TableColumn tableColumn = metadataDao.getTableColumn(tableId, temporalColumnId.getAsLong()); - if (!isValidTemporalColumn(tableId, tableColumn.getDataType())) { - return ImmutableSet.of(); - } - } - - Set filteredShards = tableShards.stream() - .filter(this::needsCompaction) - .filter(shard -> !organizer.inProgress(shard.getShardUuid())) - .collect(toSet()); - - Collection shardIndexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, false); - if (tableInfo.getTemporalColumnId().isPresent()) { - Set temporalShards = shardIndexInfos.stream() - .filter(shard -> shard.getTemporalRange().isPresent()) - .collect(toSet()); - return compactionSetCreator.createCompactionSets(tableInfo, temporalShards); - } - - return compactionSetCreator.createCompactionSets(tableInfo, shardIndexInfos); - } - - private static boolean isValidTemporalColumn(long tableId, Type type) - { - if (!type.equals(DATE) && !type.equals(TIMESTAMP_MILLIS)) { - log.warn("Temporal column type of table ID %s set incorrectly to %s", tableId, type); - return false; - } - return true; - } - - private boolean needsCompaction(ShardMetadata shard) - { - if (shard.getUncompressedSize() < (FILL_FACTOR * maxShardSize.toBytes())) { - return true; - } - - if (shard.getRowCount() < (FILL_FACTOR * maxShardRows)) { - return true; - } - return false; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactor.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactor.java deleted file mode 100644 index 76a7473a2672..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardCompactor.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; -import io.trino.orc.OrcReaderOptions; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.plugin.raptor.legacy.storage.StoragePageSink; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeOperators; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.io.Closeable; -import java.io.IOException; -import java.lang.invoke.MethodHandle; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.OptionalInt; -import java.util.PriorityQueue; -import java.util.Queue; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Throwables.throwIfUnchecked; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.BLOCK_POSITION; -import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.FAIL_ON_NULL; -import static io.trino.spi.function.InvocationConvention.simpleConvention; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public final class ShardCompactor -{ - private final StorageManager storageManager; - - private final CounterStat inputShards = new CounterStat(); - private final CounterStat outputShards = new CounterStat(); - private final DistributionStat inputShardsPerCompaction = new DistributionStat(); - private final DistributionStat outputShardsPerCompaction = new DistributionStat(); - private final DistributionStat compactionLatencyMillis = new DistributionStat(); - private final DistributionStat sortedCompactionLatencyMillis = new DistributionStat(); - private final OrcReaderOptions orcReaderOptions; - private final TypeOperators typeOperators; - - @Inject - public ShardCompactor(StorageManager storageManager, StorageManagerConfig config, TypeManager typeManager) - { - this(storageManager, - config.toOrcReaderOptions(), - typeManager.getTypeOperators()); - } - - public ShardCompactor(StorageManager storageManager, OrcReaderOptions orcReaderOptions, TypeOperators typeOperators) - { - this.storageManager = requireNonNull(storageManager, "storageManager is null"); - this.orcReaderOptions = requireNonNull(orcReaderOptions, "orcReaderOptions is null"); - this.typeOperators = requireNonNull(typeOperators, "typeOperators is null"); - } - - public List compact(long transactionId, OptionalInt bucketNumber, Set uuids, List columns) - throws IOException - { - long start = System.nanoTime(); - List columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList()); - List columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList()); - - StoragePageSink storagePageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false); - - List shardInfos; - try { - shardInfos = compact(storagePageSink, bucketNumber, uuids, columnIds, columnTypes); - } - catch (IOException | RuntimeException e) { - storagePageSink.rollback(); - throw e; - } - - updateStats(uuids.size(), shardInfos.size(), nanosSince(start).toMillis()); - return shardInfos; - } - - private List compact(StoragePageSink storagePageSink, OptionalInt bucketNumber, Set uuids, List columnIds, List columnTypes) - throws IOException - { - for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), orcReaderOptions)) { - while (!pageSource.isFinished()) { - Page page = pageSource.getNextPage(); - if (isNullOrEmptyPage(page)) { - continue; - } - storagePageSink.appendPage(page); - if (storagePageSink.isFull()) { - storagePageSink.flush(); - } - } - } - } - return getFutureValue(storagePageSink.commit()); - } - - public List compactSorted(long transactionId, OptionalInt bucketNumber, Set uuids, List columns, List sortColumnIds, List sortOrders) - throws IOException - { - checkArgument(sortColumnIds.size() == sortOrders.size(), "sortColumnIds and sortOrders must be of the same size"); - - long start = System.nanoTime(); - - List columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList()); - List columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList()); - - checkArgument(ImmutableSet.copyOf(columnIds).containsAll(sortColumnIds), "sortColumnIds must be a subset of columnIds"); - - List sortIndexes = sortColumnIds.stream() - .map(columnIds::indexOf) - .collect(toList()); - - Queue rowSources = new PriorityQueue<>(); - StoragePageSink outputPageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false); - PageBuilder pageBuilder = new PageBuilder(columnTypes); - try { - for (UUID uuid : uuids) { - ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), orcReaderOptions); - SortedRowSource rowSource = new SortedRowSource(pageSource, columnTypes, sortIndexes, sortOrders, typeOperators); - rowSources.add(rowSource); - } - while (!rowSources.isEmpty()) { - SortedRowSource rowSource = rowSources.poll(); - if (!rowSource.hasNext()) { - // rowSource is empty, close it - rowSource.close(); - continue; - } - - rowSource.next().appendTo(pageBuilder); - - if (pageBuilder.isFull()) { - outputPageSink.appendPage(pageBuilder.build()); - pageBuilder.reset(); - } - - if (outputPageSink.isFull()) { - outputPageSink.flush(); - } - - rowSources.add(rowSource); - } - - if (!pageBuilder.isEmpty()) { - outputPageSink.appendPage(pageBuilder.build()); - } - - outputPageSink.flush(); - List shardInfos = getFutureValue(outputPageSink.commit()); - - updateStats(uuids.size(), shardInfos.size(), nanosSince(start).toMillis()); - - return shardInfos; - } - catch (IOException | RuntimeException e) { - outputPageSink.rollback(); - throw e; - } - finally { - rowSources.forEach(SortedRowSource::closeQuietly); - } - } - - private static class SortedRowSource - implements Iterator, Comparable, Closeable - { - private final ConnectorPageSource pageSource; - private final List sortIndexes; - private final List orderingOperators; - - private Page currentPage; - private int currentPosition; - - public SortedRowSource(ConnectorPageSource pageSource, List columnTypes, List sortIndexes, List sortOrders, TypeOperators typeOperators) - { - this.pageSource = requireNonNull(pageSource, "pageSource is null"); - this.sortIndexes = ImmutableList.copyOf(requireNonNull(sortIndexes, "sortIndexes is null")); - requireNonNull(columnTypes, "columnTypes is null"); - requireNonNull(sortOrders, "sortOrders is null"); - - ImmutableList.Builder orderingOperators = ImmutableList.builder(); - for (int index = 0; index < sortIndexes.size(); index++) { - Type type = columnTypes.get(sortIndexes.get(index)); - SortOrder sortOrder = sortOrders.get(index); - orderingOperators.add(typeOperators.getOrderingOperator(type, sortOrder, simpleConvention(FAIL_ON_NULL, BLOCK_POSITION, BLOCK_POSITION))); - } - this.orderingOperators = orderingOperators.build(); - - currentPage = pageSource.getNextPage(); - currentPosition = 0; - } - - @Override - public boolean hasNext() - { - if (hasMorePositions(currentPage, currentPosition)) { - return true; - } - - Page page = getNextPage(pageSource); - if (isNullOrEmptyPage(page)) { - return false; - } - currentPage = page.getLoadedPage(); - currentPosition = 0; - return true; - } - - private static Page getNextPage(ConnectorPageSource pageSource) - { - Page page = null; - while (isNullOrEmptyPage(page) && !pageSource.isFinished()) { - page = pageSource.getNextPage(); - if (page != null) { - page = page.getLoadedPage(); - } - } - return page; - } - - @Override - public Row next() - { - if (!hasNext()) { - throw new NoSuchElementException(); - } - - Row row = new Row(currentPage, currentPosition); - currentPosition++; - return row; - } - - @Override - public int compareTo(SortedRowSource other) - { - if (!hasNext()) { - return 1; - } - - if (!other.hasNext()) { - return -1; - } - - try { - for (int i = 0; i < sortIndexes.size(); i++) { - int channel = sortIndexes.get(i); - - Block leftBlock = currentPage.getBlock(channel); - int leftBlockPosition = currentPosition; - - Block rightBlock = other.currentPage.getBlock(channel); - int rightBlockPosition = other.currentPosition; - - MethodHandle comparator = orderingOperators.get(i); - int compare = (int) comparator.invokeExact(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition); - if (compare != 0) { - return compare; - } - } - return 0; - } - catch (Throwable throwable) { - throwIfUnchecked(throwable); - throw new TrinoException(GENERIC_INTERNAL_ERROR, throwable); - } - } - - private static boolean hasMorePositions(Page currentPage, int currentPosition) - { - return currentPage != null && currentPosition < currentPage.getPositionCount(); - } - - void closeQuietly() - { - try { - close(); - } - catch (IOException _) { - } - } - - @Override - public void close() - throws IOException - { - pageSource.close(); - } - } - - private static class Row - { - private final Page page; - private final int position; - - public Row(Page page, int position) - { - this.page = requireNonNull(page, "page is null"); - this.position = position; - } - - public void appendTo(PageBuilder pageBuilder) - { - pageBuilder.declarePosition(); - for (int channel = 0; channel < page.getChannelCount(); channel++) { - Block block = page.getBlock(channel); - BlockBuilder output = pageBuilder.getBlockBuilder(channel); - pageBuilder.getType(channel).appendTo(block, position, output); - } - } - } - - private static boolean isNullOrEmptyPage(Page nextPage) - { - return nextPage == null || nextPage.getPositionCount() == 0; - } - - private void updateStats(int inputShardsCount, int outputShardsCount, long latency) - { - inputShards.update(inputShardsCount); - outputShards.update(outputShardsCount); - - inputShardsPerCompaction.add(inputShardsCount); - outputShardsPerCompaction.add(outputShardsCount); - - compactionLatencyMillis.add(latency); - } - - @Managed - @Nested - public CounterStat getInputShards() - { - return inputShards; - } - - @Managed - @Nested - public CounterStat getOutputShards() - { - return outputShards; - } - - @Managed - @Nested - public DistributionStat getInputShardsPerCompaction() - { - return inputShardsPerCompaction; - } - - @Managed - @Nested - public DistributionStat getOutputShardsPerCompaction() - { - return outputShardsPerCompaction; - } - - @Managed - @Nested - public DistributionStat getCompactionLatencyMillis() - { - return compactionLatencyMillis; - } - - @Managed - @Nested - public DistributionStat getSortedCompactionLatencyMillis() - { - return sortedCompactionLatencyMillis; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardIndexInfo.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardIndexInfo.java deleted file mode 100644 index cf7d03a01dda..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardIndexInfo.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardIndexInfo -{ - private final long tableId; - private final OptionalInt bucketNumber; - private final UUID shardUuid; - private final long rowCount; - private final long uncompressedSize; - private final Optional sortRange; - private final Optional temporalRange; - - public ShardIndexInfo( - long tableId, - OptionalInt bucketNumber, - UUID shardUuid, - long rowCount, - long uncompressedSize, - Optional sortRange, - Optional temporalRange) - { - this.tableId = tableId; - this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null"); - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.rowCount = rowCount; - this.uncompressedSize = uncompressedSize; - this.sortRange = requireNonNull(sortRange, "sortRange is null"); - this.temporalRange = requireNonNull(temporalRange, "temporalRange is null"); - } - - public long getTableId() - { - return tableId; - } - - public OptionalInt getBucketNumber() - { - return bucketNumber; - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public long getRowCount() - { - return rowCount; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - - public Optional getSortRange() - { - return sortRange; - } - - public Optional getTemporalRange() - { - return temporalRange; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ShardIndexInfo that = (ShardIndexInfo) o; - return tableId == that.tableId && - rowCount == that.rowCount && - uncompressedSize == that.uncompressedSize && - Objects.equals(bucketNumber, that.bucketNumber) && - Objects.equals(shardUuid, that.shardUuid) && - Objects.equals(sortRange, that.sortRange) && - Objects.equals(temporalRange, that.temporalRange); - } - - @Override - public int hashCode() - { - return Objects.hash(tableId, bucketNumber, shardUuid, rowCount, uncompressedSize, sortRange, temporalRange); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("tableId", tableId) - .add("bucketNumber", bucketNumber.isPresent() ? bucketNumber.getAsInt() : null) - .add("shardUuid", shardUuid) - .add("rowCount", rowCount) - .add("uncompressedSize", uncompressedSize) - .add("sortRange", sortRange.orElse(null)) - .add("temporalRange", temporalRange.orElse(null)) - .omitNullValues() - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizationManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizationManager.java deleted file mode 100644 index cb6e1792506f..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizationManager.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.NodeManager; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.jdbi.v3.core.Jdbi; - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.collect.Sets.difference; -import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.createOrganizationSet; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getOrganizationEligibleShards; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getShardsByDaysBuckets; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static java.lang.Math.max; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; - -public class ShardOrganizationManager -{ - private static final Logger log = Logger.get(ShardOrganizationManager.class); - - private final ScheduledExecutorService discoveryService = newScheduledThreadPool(1, daemonThreadsNamed("shard-organization-discovery")); - private final AtomicBoolean started = new AtomicBoolean(); - - private final Jdbi dbi; - private final MetadataDao metadataDao; - private final ShardOrganizerDao organizerDao; - private final ShardManager shardManager; - - private final boolean enabled; - private final long organizationIntervalMillis; - private final long organizationDiscoveryIntervalMillis; - - private final String currentNodeIdentifier; - private final ShardOrganizer organizer; - - private final Set tablesInProgress = newConcurrentHashSet(); - - @Inject - public ShardOrganizationManager( - @ForMetadata Jdbi dbi, - NodeManager nodeManager, - ShardManager shardManager, - ShardOrganizer organizer, - StorageManagerConfig config) - { - this(dbi, - nodeManager.getCurrentNode().getNodeIdentifier(), - shardManager, - organizer, - config.isOrganizationEnabled(), - config.getOrganizationInterval(), - config.getOrganizationDiscoveryInterval()); - } - - public ShardOrganizationManager( - Jdbi dbi, - String currentNodeIdentifier, - ShardManager shardManager, - ShardOrganizer organizer, - boolean enabled, - Duration organizationInterval, - Duration organizationDiscoveryInterval) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.metadataDao = onDemandDao(dbi, MetadataDao.class); - this.organizerDao = onDemandDao(dbi, ShardOrganizerDao.class); - - this.organizer = requireNonNull(organizer, "organizer is null"); - this.shardManager = requireNonNull(shardManager, "shardManager is null"); - this.currentNodeIdentifier = requireNonNull(currentNodeIdentifier, "currentNodeIdentifier is null"); - - this.enabled = enabled; - - requireNonNull(organizationInterval, "organizationInterval is null"); - this.organizationIntervalMillis = max(1, organizationInterval.roundTo(MILLISECONDS)); - this.organizationDiscoveryIntervalMillis = max(1, organizationDiscoveryInterval.roundTo(MILLISECONDS)); - } - - @PostConstruct - public void start() - { - if (!enabled || started.getAndSet(true)) { - return; - } - - startDiscovery(); - } - - @PreDestroy - public void shutdown() - { - discoveryService.shutdownNow(); - } - - private void startDiscovery() - { - discoveryService.scheduleWithFixedDelay(() -> { - try { - // jitter to avoid overloading database and overloading the backup store - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, organizationDiscoveryIntervalMillis)); - - log.info("Running shard organizer..."); - submitJobs(discoverAndInitializeTablesToOrganize()); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable t) { - log.error(t, "Error running shard organizer"); - } - }, 0, organizationDiscoveryIntervalMillis, TimeUnit.MILLISECONDS); - } - - @VisibleForTesting - Set discoverAndInitializeTablesToOrganize() - { - Set enabledTableIds = metadataDao.getOrganizationEligibleTables(); - - Set tableOrganizationInfo = organizerDao.getNodeTableOrganizationInfo(currentNodeIdentifier); - Map organizationInfos = Maps.uniqueIndex(tableOrganizationInfo, TableOrganizationInfo::getTableId); - - // If this is the first time organizing a table, initialize the organization info for it - difference(enabledTableIds, organizationInfos.keySet()) - .forEach(tableId -> organizerDao.insertNode(currentNodeIdentifier, tableId)); - - ImmutableSet.Builder tableIds = ImmutableSet.builder(); - for (Long tableId : enabledTableIds) { - TableOrganizationInfo info = organizationInfos.get(tableId); - if (info == null || shouldRunOrganization(info)) { - tableIds.add(tableId); - } - } - return tableIds.build(); - } - - private void submitJobs(Set tableIds) - { - tableIds.forEach(this::runOrganization); - } - - private void runOrganization(long tableId) - { - Set shardMetadatas = shardManager.getNodeShards(currentNodeIdentifier, tableId); - Table tableInfo = metadataDao.getTableInformation(tableId); - Set filteredShards = shardMetadatas.stream() - .filter(shard -> !organizer.inProgress(shard.getShardUuid())) - .collect(toSet()); - - Collection indexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, true); - Set organizationSets = createOrganizationSets(tableInfo, indexInfos); - - if (organizationSets.isEmpty()) { - return; - } - - log.info("Created %s organization set(s) from %s shards for table ID %s", organizationSets.size(), filteredShards.size(), tableId); - - long lastStartTime = System.currentTimeMillis(); - tablesInProgress.add(tableId); - - ImmutableList.Builder> futures = ImmutableList.builder(); - for (OrganizationSet organizationSet : organizationSets) { - futures.add(organizer.enqueue(organizationSet)); - } - allAsList(futures.build()) - .whenComplete((value, throwable) -> { - tablesInProgress.remove(tableId); - organizerDao.updateLastStartTime(currentNodeIdentifier, tableId, lastStartTime); - }); - } - - private boolean shouldRunOrganization(TableOrganizationInfo info) - { - // skip if organization is in progress for this table - if (tablesInProgress.contains(info.getTableId())) { - return false; - } - - if (info.getLastStartTimeMillis().isEmpty()) { - return true; - } - - return (System.currentTimeMillis() - info.getLastStartTimeMillis().getAsLong()) >= organizationIntervalMillis; - } - - @VisibleForTesting - static Set createOrganizationSets(Table tableInfo, Collection shards) - { - return getShardsByDaysBuckets(tableInfo, shards).stream() - .map(indexInfos -> getOverlappingOrganizationSets(tableInfo, indexInfos)) - .flatMap(Collection::stream) - .collect(toSet()); - } - - private static Set getOverlappingOrganizationSets(Table tableInfo, Collection shards) - { - if (shards.size() <= 1) { - return ImmutableSet.of(); - } - - // Sort by low marker for the range - List sortedShards = shards.stream() - .sorted((o1, o2) -> { - ShardRange sortRange1 = o1.getSortRange().get(); - ShardRange sortRange2 = o2.getSortRange().get(); - return ComparisonChain.start() - .compare(sortRange1.getMinTuple(), sortRange2.getMinTuple()) - .compare(sortRange2.getMaxTuple(), sortRange1.getMaxTuple()) - .result(); - }) - .collect(toList()); - - Set organizationSets = new HashSet<>(); - ImmutableSet.Builder builder = ImmutableSet.builder(); - - builder.add(sortedShards.get(0)); - int previousRange = 0; - int nextRange = previousRange + 1; - while (nextRange < sortedShards.size()) { - ShardRange sortRange1 = sortedShards.get(previousRange).getSortRange().get(); - ShardRange sortRange2 = sortedShards.get(nextRange).getSortRange().get(); - - if (sortRange1.overlaps(sortRange2) && !sortRange1.adjacent(sortRange2)) { - builder.add(sortedShards.get(nextRange)); - if (!sortRange1.encloses(sortRange2)) { - previousRange = nextRange; - } - } - else { - Set indexInfos = builder.build(); - if (indexInfos.size() > 1) { - organizationSets.add(createOrganizationSet(tableInfo.getTableId(), indexInfos)); - } - builder = ImmutableSet.builder(); - previousRange = nextRange; - builder.add(sortedShards.get(previousRange)); - } - nextRange++; - } - - Set indexInfos = builder.build(); - if (indexInfos.size() > 1) { - organizationSets.add(createOrganizationSet(tableInfo.getTableId(), indexInfos)); - } - return organizationSets; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizer.java deleted file mode 100644 index ba9512c9a027..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizer.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.inject.Inject; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import jakarta.annotation.PreDestroy; -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadPoolExecutor; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Sets.newConcurrentHashSet; -import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.runAsync; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class ShardOrganizer -{ - private static final Logger log = Logger.get(ShardOrganizer.class); - - private final ExecutorService executorService; - private final ThreadPoolExecutorMBean executorMBean; - - // Tracks shards that are scheduled for compaction so that we do not schedule them more than once - private final Set shardsInProgress = newConcurrentHashSet(); - private final JobFactory jobFactory; - private final CounterStat successCount = new CounterStat(); - private final CounterStat failureCount = new CounterStat(); - - @Inject - public ShardOrganizer(JobFactory jobFactory, StorageManagerConfig config) - { - this(jobFactory, config.getOrganizationThreads()); - } - - public ShardOrganizer(JobFactory jobFactory, int threads) - { - checkArgument(threads > 0, "threads must be > 0"); - this.jobFactory = requireNonNull(jobFactory, "jobFactory is null"); - this.executorService = newFixedThreadPool(threads, daemonThreadsNamed("shard-organizer-%s")); - this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) executorService); - } - - @PreDestroy - public void shutdown() - { - shutdownAndAwaitTermination(executorService, 10, SECONDS); - } - - public CompletableFuture enqueue(OrganizationSet organizationSet) - { - shardsInProgress.addAll(organizationSet.getShards()); - return runAsync(jobFactory.create(organizationSet), executorService) - .whenComplete((none, throwable) -> { - shardsInProgress.removeAll(organizationSet.getShards()); - if (throwable == null) { - successCount.update(1); - } - else { - log.warn(throwable, "Error running organization job"); - failureCount.update(1); - } - }); - } - - public boolean inProgress(UUID shardUuid) - { - return shardsInProgress.contains(shardUuid); - } - - @Managed - @Nested - public ThreadPoolExecutorMBean getExecutor() - { - return executorMBean; - } - - @Managed - public int getShardsInProgress() - { - return shardsInProgress.size(); - } - - @Managed - @Nested - public CounterStat getSuccessCount() - { - return successCount; - } - - @Managed - @Nested - public CounterStat getFailureCount() - { - return failureCount; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerDao.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerDao.java deleted file mode 100644 index 7acd59874e4e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerDao.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.util.Set; - -@RegisterConstructorMapper(TableOrganizationInfo.class) -public interface ShardOrganizerDao -{ - @SqlUpdate("INSERT INTO shard_organizer_jobs (node_identifier, table_id, last_start_time)\n" + - "VALUES (:nodeIdentifier, :tableId, NULL)") - void insertNode(String nodeIdentifier, long tableId); - - @SqlUpdate("UPDATE shard_organizer_jobs SET last_start_time = :lastStartTime\n" + - " WHERE node_identifier = :nodeIdentifier\n" + - " AND table_id = :tableId") - void updateLastStartTime( - String nodeIdentifier, - long tableId, - long lastStartTime); - - @SqlQuery("SELECT table_id, last_start_time\n" + - " FROM shard_organizer_jobs\n" + - " WHERE node_identifier = :nodeIdentifier") - Set getNodeTableOrganizationInfo(String nodeIdentifier); - - @SqlUpdate("DELETE FROM shard_organizer_jobs WHERE table_id = :tableId") - void dropOrganizerJobs(long tableId); -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerUtil.java deleted file mode 100644 index ae26b616a553..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardOrganizerUtil.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Multimaps; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Jdbi; - -import java.sql.Connection; -import java.sql.JDBCType; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.collect.Iterables.partition; -import static com.google.common.collect.Maps.uniqueIndex; -import static io.airlift.slice.Slices.wrappedBuffer; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.storage.ColumnIndexStatsUtils.jdbcType; -import static java.lang.String.format; -import static java.util.Collections.nCopies; -import static java.util.stream.Collectors.toSet; - -public final class ShardOrganizerUtil -{ - private ShardOrganizerUtil() {} - - public static Collection getOrganizationEligibleShards( - Jdbi dbi, - MetadataDao metadataDao, - Table tableInfo, - Collection shards, - boolean includeSortColumns) - { - Map shardsById = uniqueIndex(shards, ShardMetadata::getShardId); - long tableId = tableInfo.getTableId(); - - ImmutableList.Builder columnsBuilder = ImmutableList.builder(); - columnsBuilder.add("shard_id"); - - // include temporal columns if present - Optional temporalColumn = Optional.empty(); - if (tableInfo.getTemporalColumnId().isPresent()) { - long temporalColumnId = tableInfo.getTemporalColumnId().getAsLong(); - temporalColumn = Optional.of(metadataDao.getTableColumn(tableId, temporalColumnId)); - columnsBuilder.add(minColumn(temporalColumnId), maxColumn(temporalColumnId)); - } - - // include sort columns if needed - Optional> sortColumns = Optional.empty(); - if (includeSortColumns) { - sortColumns = Optional.of(metadataDao.listSortColumns(tableId)); - for (TableColumn column : sortColumns.get()) { - columnsBuilder.add(minColumn(column.getColumnId()), maxColumn(column.getColumnId())); - } - } - String columnToSelect = Joiner.on(",\n").join(columnsBuilder.build()); - - ImmutableList.Builder indexInfoBuilder = ImmutableList.builder(); - try (Connection connection = dbi.open().getConnection()) { - for (List partitionedShards : partition(shards, 1000)) { - String shardIds = Joiner.on(",").join(nCopies(partitionedShards.size(), "?")); - - String sql = format("" + - "SELECT %s\n" + - "FROM %s\n" + - "WHERE shard_id IN (%s)", - columnToSelect, shardIndexTable(tableId), shardIds); - - try (PreparedStatement statement = connection.prepareStatement(sql)) { - for (int i = 0; i < partitionedShards.size(); i++) { - statement.setLong(i + 1, partitionedShards.get(i).getShardId()); - } - try (ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - long shardId = resultSet.getLong("shard_id"); - - Optional sortRange = Optional.empty(); - if (includeSortColumns) { - sortRange = getShardRange(sortColumns.get(), resultSet); - if (sortRange.isEmpty()) { - continue; - } - } - Optional temporalRange = Optional.empty(); - if (temporalColumn.isPresent()) { - temporalRange = getShardRange(ImmutableList.of(temporalColumn.get()), resultSet); - if (temporalRange.isEmpty()) { - continue; - } - } - ShardMetadata shardMetadata = shardsById.get(shardId); - indexInfoBuilder.add(toShardIndexInfo(shardMetadata, temporalRange, sortRange)); - } - } - } - } - } - catch (SQLException e) { - throw new RuntimeException(e); - } - return indexInfoBuilder.build(); - } - - private static ShardIndexInfo toShardIndexInfo(ShardMetadata shardMetadata, Optional temporalRange, Optional sortRange) - { - return new ShardIndexInfo( - shardMetadata.getTableId(), - shardMetadata.getBucketNumber(), - shardMetadata.getShardUuid(), - shardMetadata.getRowCount(), - shardMetadata.getUncompressedSize(), - sortRange, - temporalRange); - } - - public static Collection> getShardsByDaysBuckets(Table tableInfo, Collection shards) - { - if (shards.isEmpty()) { - return ImmutableList.of(); - } - - // Neither bucketed nor temporal, no partitioning required - if (tableInfo.getBucketCount().isEmpty() && tableInfo.getTemporalColumnId().isEmpty()) { - return ImmutableList.of(shards); - } - - // if only bucketed, partition by bucket number - if (tableInfo.getBucketCount().isPresent() && tableInfo.getTemporalColumnId().isEmpty()) { - return Multimaps.index(shards, shard -> shard.getBucketNumber().getAsInt()).asMap().values(); - } - - // if temporal, partition into days first - ImmutableMultimap.Builder shardsByDaysBuilder = ImmutableMultimap.builder(); - shards.stream() - .filter(shard -> shard.getTemporalRange().isPresent()) - .forEach(shard -> { - long day = TemporalFunction.getDayFromRange(shard.getTemporalRange().get()); - shardsByDaysBuilder.put(day, shard); - }); - - Collection> byDays = shardsByDaysBuilder.build().asMap().values(); - - // if table is bucketed further partition by bucket number - if (tableInfo.getBucketCount().isEmpty()) { - return byDays; - } - - ImmutableList.Builder> sets = ImmutableList.builder(); - for (Collection s : byDays) { - sets.addAll(Multimaps.index(s, ShardIndexInfo::getBucketNumber).asMap().values()); - } - return sets.build(); - } - - private static Optional getShardRange(List columns, ResultSet resultSet) - throws SQLException - { - ImmutableList.Builder minValuesBuilder = ImmutableList.builder(); - ImmutableList.Builder maxValuesBuilder = ImmutableList.builder(); - ImmutableList.Builder typeBuilder = ImmutableList.builder(); - - for (TableColumn tableColumn : columns) { - long columnId = tableColumn.getColumnId(); - Type type = tableColumn.getDataType(); - - Object min = getValue(resultSet, type, minColumn(columnId)); - Object max = getValue(resultSet, type, maxColumn(columnId)); - - if (min == null || max == null) { - return Optional.empty(); - } - - minValuesBuilder.add(min); - maxValuesBuilder.add(max); - typeBuilder.add(type); - } - - List types = typeBuilder.build(); - Tuple minTuple = new Tuple(types, minValuesBuilder.build()); - Tuple maxTuple = new Tuple(types, maxValuesBuilder.build()); - - return Optional.of(ShardRange.of(minTuple, maxTuple)); - } - - private static Object getValue(ResultSet resultSet, Type type, String columnName) - throws SQLException - { - JDBCType jdbcType = jdbcType(type); - Object value = getValue(resultSet, type, columnName, jdbcType); - return resultSet.wasNull() ? null : value; - } - - private static Object getValue(ResultSet resultSet, Type type, String columnName, JDBCType jdbcType) - throws SQLException - { - return switch (jdbcType) { - case BOOLEAN -> resultSet.getBoolean(columnName); - case INTEGER -> resultSet.getInt(columnName); - case BIGINT -> resultSet.getLong(columnName); - case DOUBLE -> resultSet.getDouble(columnName); - case VARBINARY -> wrappedBuffer(resultSet.getBytes(columnName)).toStringUtf8(); - default -> throw new IllegalArgumentException("Unhandled type: " + type); - }; - } - - static OrganizationSet createOrganizationSet(long tableId, Set shardsToCompact) - { - Set uuids = shardsToCompact.stream() - .map(ShardIndexInfo::getShardUuid) - .collect(toSet()); - - Set bucketNumber = shardsToCompact.stream() - .map(ShardIndexInfo::getBucketNumber) - .collect(toSet()); - - checkArgument(bucketNumber.size() == 1); - return new OrganizationSet(tableId, uuids, getOnlyElement(bucketNumber)); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardRange.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardRange.java deleted file mode 100644 index eec709716039..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/ShardRange.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardRange -{ - private final Tuple minTuple; - private final Tuple maxTuple; - - public static ShardRange of(Tuple min, Tuple max) - { - return new ShardRange(min, max); - } - - private ShardRange(Tuple minTuple, Tuple maxTuple) - { - this.minTuple = requireNonNull(minTuple, "minTuple is null"); - this.maxTuple = requireNonNull(maxTuple, "maxTuple is null"); - } - - public Tuple getMinTuple() - { - return minTuple; - } - - public Tuple getMaxTuple() - { - return maxTuple; - } - - public boolean encloses(ShardRange other) - { - return this.getMinTuple().compareTo(other.getMinTuple()) <= 0 && - this.getMaxTuple().compareTo(other.getMaxTuple()) >= 0; - } - - public boolean overlaps(ShardRange other) - { - return this.getMinTuple().compareTo(other.getMaxTuple()) <= 0 && - other.getMinTuple().compareTo(this.getMaxTuple()) <= 0; - } - - public boolean adjacent(ShardRange other) - { - Object o1Min = this.getMinTuple().getValues().get(0); - Object o1Max = this.getMaxTuple().getValues().get(0); - - Object o2Min = other.getMinTuple().getValues().get(0); - Object o2Max = other.getMaxTuple().getValues().get(0); - - return o1Max.equals(o2Min) || o2Max.equals(o1Min); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ShardRange that = (ShardRange) o; - return Objects.equals(minTuple, that.minTuple) && - Objects.equals(maxTuple, that.maxTuple); - } - - @Override - public int hashCode() - { - return Objects.hash(minTuple, maxTuple); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("minTuple", minTuple) - .add("maxTuple", maxTuple) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TableOrganizationInfo.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TableOrganizationInfo.java deleted file mode 100644 index 907e4c0b7ae1..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TableOrganizationInfo.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import org.jdbi.v3.core.mapper.reflect.ColumnName; - -import java.util.OptionalLong; - -import static java.util.Objects.requireNonNull; - -public class TableOrganizationInfo -{ - private final long tableId; - private final OptionalLong lastStartTimeMillis; - - public TableOrganizationInfo(long tableId, @ColumnName("last_start_time") OptionalLong lastStartTimeMillis) - { - this.tableId = tableId; - this.lastStartTimeMillis = requireNonNull(lastStartTimeMillis, "lastStartTimeMillis is null"); - } - - public long getTableId() - { - return tableId; - } - - public OptionalLong getLastStartTimeMillis() - { - return lastStartTimeMillis; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TemporalFunction.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TemporalFunction.java deleted file mode 100644 index 94c35acdce91..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/TemporalFunction.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import io.trino.spi.block.Block; -import io.trino.spi.type.Type; - -import java.time.Duration; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_DAY; -import static java.lang.Math.floorDiv; -import static java.lang.Math.toIntExact; - -public final class TemporalFunction -{ - private TemporalFunction() {} - - public static int getDay(Type type, Block block, int position) - { - if (type.equals(DATE)) { - return DATE.getInt(block, position); - } - - if (type.equals(TIMESTAMP_MILLIS)) { - long days = floorDiv(TIMESTAMP_MILLIS.getLong(block, position), MICROSECONDS_PER_DAY); - return toIntExact(days); - } - - throw new IllegalArgumentException("Wrong type for temporal column: " + type); - } - - public static int getDayFromRange(ShardRange range) - { - Tuple min = range.getMinTuple(); - Tuple max = range.getMaxTuple(); - checkArgument(getOnlyElement(min.getTypes()).equals(getOnlyElement(max.getTypes())), "type of min and max is not same"); - - Type type = getOnlyElement(min.getTypes()); - if (type.equals(DATE)) { - return (int) getOnlyElement(min.getValues()); - } - - if (type.equals(TIMESTAMP_MILLIS)) { - long minValue = (long) getOnlyElement(min.getValues()); - long maxValue = (long) getOnlyElement(max.getValues()); - return determineDay(minValue, maxValue); - } - - throw new IllegalArgumentException("Wrong type for shard range: " + type); - } - - private static int determineDay(long rangeStart, long rangeEnd) - { - int startDay = toIntExact(Duration.ofMillis(rangeStart).toDays()); - int endDay = toIntExact(Duration.ofMillis(rangeEnd).toDays()); - if (startDay == endDay) { - return startDay; - } - - if ((endDay - startDay) > 1) { - // range spans multiple days, return the first full day - return startDay + 1; - } - - // range spans two days, return the day that has the larger time range - long millisInStartDay = Duration.ofDays(endDay).toMillis() - rangeStart; - long millisInEndDay = rangeEnd - Duration.ofDays(endDay).toMillis(); - return (millisInStartDay >= millisInEndDay) ? startDay : endDay; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/Tuple.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/Tuple.java deleted file mode 100644 index e1db17363b56..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/organization/Tuple.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.type.Type; - -import java.util.List; -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class Tuple - implements Comparable -{ - private final List types; - private final List values; - - public Tuple(Type type, Object value) - { - this(ImmutableList.of(type), ImmutableList.of(value)); - } - - public Tuple(List types, Object... values) - { - this(types, ImmutableList.copyOf(values)); - } - - public Tuple(List types, List values) - { - this.types = requireNonNull(types, "types is null"); - this.values = requireNonNull(values, "values is null"); - - checkArgument(!types.isEmpty(), "types is empty"); - checkArgument(types.size() == values.size(), "types and values must have the same number of elements"); - } - - public List getTypes() - { - return types; - } - - public List getValues() - { - return values; - } - - @Override - public int compareTo(Tuple o) - { - checkArgument(o.getTypes().size() == types.size(), "types must be of same size"); - for (int i = 0; i < types.size(); i++) { - checkArgument(o.getTypes().get(i).equals(types.get(i)), "types must be the same"); - } - - for (int i = 0; i < types.size(); i++) { - Object o1 = values.get(i); - Object o2 = o.getValues().get(i); - - int result = compare(o1, o2); - if (result != 0) { - return result; - } - } - return 0; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Tuple tuple = (Tuple) o; - return Objects.equals(types, tuple.types) && - Objects.equals(values, tuple.values); - } - - @Override - public int hashCode() - { - return Objects.hash(types, values); - } - - @SuppressWarnings("unchecked") - private static int compare(Object o1, Object o2) - { - return ((Comparable) o1).compareTo((T) o2); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("values", values) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ColumnRangesSystemTable.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ColumnRangesSystemTable.java deleted file mode 100644 index 6838c4201567..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ColumnRangesSystemTable.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.base.VerifyException; -import io.trino.plugin.raptor.legacy.RaptorTableHandle; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.FixedPageSource; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.JdbiException; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.joining; - -public class ColumnRangesSystemTable - implements SystemTable -{ - private static final String MIN_COLUMN_SUFFIX = "_min"; - private static final String MAX_COLUMN_SUFFIX = "_max"; - private static final String COLUMN_RANGES_TABLE_SUFFIX = "$column_ranges"; - - private final Jdbi dbi; - private final RaptorTableHandle sourceTable; - private final List indexedRaptorColumns; - private final ConnectorTableMetadata tableMetadata; - - public ColumnRangesSystemTable(RaptorTableHandle sourceTable, Jdbi dbi) - { - this.sourceTable = requireNonNull(sourceTable, "sourceTable is null"); - this.dbi = requireNonNull(dbi, "dbi is null"); - - this.indexedRaptorColumns = dbi.onDemand(MetadataDao.class) - .listTableColumns(sourceTable.getTableId()).stream() - .filter(column -> isIndexedType(column.getDataType())) - .collect(toImmutableList()); - List systemTableColumns = indexedRaptorColumns.stream() - .flatMap(column -> Stream.of( - new ColumnMetadata(column.getColumnName() + MIN_COLUMN_SUFFIX, column.getDataType()), - new ColumnMetadata(column.getColumnName() + MAX_COLUMN_SUFFIX, column.getDataType()))) - .collect(toImmutableList()); - SchemaTableName tableName = new SchemaTableName(sourceTable.getSchemaName(), sourceTable.getTableName() + COLUMN_RANGES_TABLE_SUFFIX); - this.tableMetadata = new ConnectorTableMetadata(tableName, systemTableColumns); - } - - public static Optional getSourceTable(SchemaTableName tableName) - { - if (tableName.getTableName().endsWith(COLUMN_RANGES_TABLE_SUFFIX) && - !tableName.getTableName().equals(COLUMN_RANGES_TABLE_SUFFIX)) { - int tableNameLength = tableName.getTableName().length() - COLUMN_RANGES_TABLE_SUFFIX.length(); - return Optional.of(new SchemaTableName( - tableName.getSchemaName(), - tableName.getTableName().substring(0, tableNameLength))); - } - return Optional.empty(); - } - - @Override - public Distribution getDistribution() - { - return SINGLE_COORDINATOR; - } - - @Override - public ConnectorTableMetadata getTableMetadata() - { - return tableMetadata; - } - - @Override - public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) - { - String metadataSqlQuery = getColumnRangesMetadataSqlQuery(sourceTable, indexedRaptorColumns); - List columnTypes = tableMetadata.getColumns().stream() - .map(ColumnMetadata::getType) - .collect(toImmutableList()); - - PageListBuilder pageListBuilder = new PageListBuilder(columnTypes); - - try (Connection connection = dbi.open().getConnection(); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(metadataSqlQuery)) { - if (resultSet.next()) { - pageListBuilder.beginRow(); - for (int i = 0; i < columnTypes.size(); ++i) { - BlockBuilder blockBuilder = pageListBuilder.nextBlockBuilder(); - Type columnType = columnTypes.get(i); - if (columnType.equals(BIGINT) || columnType.equals(DATE)) { - long value = resultSet.getLong(i + 1); - if (!resultSet.wasNull()) { - columnType.writeLong(blockBuilder, value); - } - else { - blockBuilder.appendNull(); - } - } - else if (columnType.equals(TIMESTAMP_MILLIS)) { - long value = resultSet.getLong(i + 1); - if (!resultSet.wasNull()) { - columnType.writeLong(blockBuilder, value * MICROSECONDS_PER_MILLISECOND); - } - else { - blockBuilder.appendNull(); - } - } - else if (columnType.equals(BOOLEAN)) { - boolean value = resultSet.getBoolean(i + 1); - if (!resultSet.wasNull()) { - BOOLEAN.writeBoolean(blockBuilder, value); - } - else { - blockBuilder.appendNull(); - } - } - else { - throw new VerifyException("Unknown or unsupported column type: " + columnType); - } - } - } - } - catch (SQLException | JdbiException e) { - throw metadataError(e); - } - - return new FixedPageSource(pageListBuilder.build()); - } - - private static boolean isIndexedType(Type type) - { - // We only consider the following types in the column_ranges system table - // Exclude INTEGER because we don't collect column stats for INTEGER type. - // Exclude DOUBLE because Java double is not completely compatible with MySQL double - // Exclude VARCHAR because they can be truncated - return type.equals(BOOLEAN) || type.equals(BIGINT) || type.equals(DATE) || type.equals(TIMESTAMP_MILLIS); - } - - private static String getColumnRangesMetadataSqlQuery(RaptorTableHandle raptorTableHandle, List raptorColumns) - { - String columns = raptorColumns.stream() - .flatMap(column -> Stream.of( - format("min(%s)", minColumn(column.getColumnId())), - format("max(%s)", maxColumn(column.getColumnId())))) - .collect(joining(", ")); - return format("SELECT %s FROM %s", columns, shardIndexTable(raptorTableHandle.getTableId())); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PageListBuilder.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PageListBuilder.java deleted file mode 100644 index 76800ff382fe..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PageListBuilder.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.Type; - -import java.util.List; - -class PageListBuilder -{ - private final PageBuilder pageBuilder; - private final ImmutableList.Builder pages = ImmutableList.builder(); - private int channel; - - public PageListBuilder(List types) - { - this.pageBuilder = new PageBuilder(types); - } - - public List build() - { - if (!pageBuilder.isEmpty()) { - pages.add(pageBuilder.build()); - pageBuilder.reset(); - } - return pages.build(); - } - - public void beginRow() - { - if (pageBuilder.isFull()) { - pages.add(pageBuilder.build()); - pageBuilder.reset(); - } - pageBuilder.declarePosition(); - channel = 0; - } - - public BlockBuilder nextBlockBuilder() - { - int currentChannel = channel; - channel++; - return pageBuilder.getBlockBuilder(currentChannel); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PreparedStatementBuilder.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PreparedStatementBuilder.java deleted file mode 100644 index 1230ef90bc1d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/PreparedStatementBuilder.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Strings.isNullOrEmpty; -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.enableStreamingResults; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidToBytes; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static java.lang.String.format; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.sql.ResultSet.CONCUR_READ_ONLY; -import static java.sql.ResultSet.TYPE_FORWARD_ONLY; -import static java.util.Collections.nCopies; -import static java.util.UUID.fromString; - -public final class PreparedStatementBuilder -{ - private PreparedStatementBuilder() {} - - public static PreparedStatement create( - Connection connection, - String sql, - List columnNames, - List types, - Set uuidColumnIndexes, - TupleDomain tupleDomain) - throws SQLException - { - checkArgument(!isNullOrEmpty(sql), "sql is null or empty"); - - List bindValues = new ArrayList<>(256); - sql += getWhereClause(tupleDomain, columnNames, types, uuidColumnIndexes, bindValues); - - PreparedStatement statement = connection.prepareStatement(sql, TYPE_FORWARD_ONLY, CONCUR_READ_ONLY); - enableStreamingResults(statement); - - // bind values to statement - int bindIndex = 1; - for (ValueBuffer value : bindValues) { - bindField(value, statement, bindIndex, uuidColumnIndexes.contains(value.getColumnIndex())); - bindIndex++; - } - return statement; - } - - @SuppressWarnings("OptionalGetWithoutIsPresent") - private static String getWhereClause( - TupleDomain tupleDomain, - List columnNames, - List types, - Set uuidColumnIndexes, - List bindValues) - { - if (tupleDomain.isNone()) { - return ""; - } - - ImmutableList.Builder conjunctsBuilder = ImmutableList.builder(); - Map domainMap = tupleDomain.getDomains().get(); - for (Map.Entry entry : domainMap.entrySet()) { - int index = entry.getKey(); - String columnName = columnNames.get(index); - Type type = types.get(index); - conjunctsBuilder.add(toPredicate(index, columnName, type, entry.getValue(), uuidColumnIndexes, bindValues)); - } - List conjuncts = conjunctsBuilder.build(); - - if (conjuncts.isEmpty()) { - return ""; - } - StringBuilder where = new StringBuilder("WHERE "); - return Joiner.on(" AND\n").appendTo(where, conjuncts).toString(); - } - - private static String toPredicate( - int columnIndex, - String columnName, - Type type, - Domain domain, - Set uuidColumnIndexes, - List bindValues) - { - if (domain.getValues().isAll()) { - return domain.isNullAllowed() ? "TRUE" : columnName + " IS NOT NULL"; - } - if (domain.getValues().isNone()) { - return domain.isNullAllowed() ? columnName + " IS NULL" : "FALSE"; - } - - return domain.getValues().getValuesProcessor().transform( - ranges -> { - // Add disjuncts for ranges - List disjuncts = new ArrayList<>(); - List singleValues = new ArrayList<>(); - - // Add disjuncts for ranges - for (Range range : ranges.getOrderedRanges()) { - checkState(!range.isAll()); // Already checked - if (range.isSingleValue()) { - singleValues.add(range.getSingleValue()); - } - else { - List rangeConjuncts = new ArrayList<>(); - if (!range.isLowUnbounded()) { - Object bindValue = getBindValue(columnIndex, uuidColumnIndexes, range.getLowBoundedValue()); - rangeConjuncts.add(toBindPredicate(columnName, range.isLowInclusive() ? ">=" : ">")); - bindValues.add(ValueBuffer.create(columnIndex, type, bindValue)); - } - if (!range.isHighUnbounded()) { - Object bindValue = getBindValue(columnIndex, uuidColumnIndexes, range.getHighBoundedValue()); - rangeConjuncts.add(toBindPredicate(columnName, range.isHighInclusive() ? "<=" : "<")); - bindValues.add(ValueBuffer.create(columnIndex, type, bindValue)); - } - // If rangeConjuncts is null, then the range was ALL, which should already have been checked for - checkState(!rangeConjuncts.isEmpty()); - disjuncts.add("(" + Joiner.on(" AND ").join(rangeConjuncts) + ")"); - } - } - - // Add back all of the possible single values either as an equality or an IN predicate - if (singleValues.size() == 1) { - disjuncts.add(toBindPredicate(columnName, "=")); - bindValues.add(ValueBuffer.create(columnIndex, type, getBindValue(columnIndex, uuidColumnIndexes, getOnlyElement(singleValues)))); - } - else if (singleValues.size() > 1) { - disjuncts.add(columnName + " IN (" + Joiner.on(",").join(nCopies(singleValues.size(), "?")) + ")"); - for (Object singleValue : singleValues) { - bindValues.add(ValueBuffer.create(columnIndex, type, getBindValue(columnIndex, uuidColumnIndexes, singleValue))); - } - } - - // Add nullability disjuncts - checkState(!disjuncts.isEmpty()); - if (domain.isNullAllowed()) { - disjuncts.add(columnName + " IS NULL"); - } - - return "(" + Joiner.on(" OR ").join(disjuncts) + ")"; - }, - - discreteValues -> { - String values = Joiner.on(",").join(nCopies(discreteValues.getValues().size(), "?")); - String predicate = columnName + (discreteValues.isInclusive() ? "" : " NOT") + " IN (" + values + ")"; - for (Object value : discreteValues.getValues()) { - bindValues.add(ValueBuffer.create(columnIndex, type, getBindValue(columnIndex, uuidColumnIndexes, value))); - } - if (domain.isNullAllowed()) { - predicate = "(" + predicate + " OR " + columnName + " IS NULL)"; - } - return predicate; - }, - - allOrNone -> { - throw new IllegalStateException("Case should not be reachable"); - }); - } - - private static Object getBindValue(int columnIndex, Set uuidColumnIndexes, Object value) - { - if (uuidColumnIndexes.contains(columnIndex)) { - return uuidToBytes(fromString(((Slice) value).toStringUtf8())); - } - return value; - } - - private static String toBindPredicate(String columnName, String operator) - { - return format("%s %s ?", columnName, operator); - } - - private static void bindField(ValueBuffer valueBuffer, PreparedStatement preparedStatement, int parameterIndex, boolean isUuid) - throws SQLException - { - Type type = valueBuffer.getType(); - if (valueBuffer.isNull()) { - preparedStatement.setNull(parameterIndex, typeToSqlType(type)); - } - else if (type.getJavaType() == long.class) { - preparedStatement.setLong(parameterIndex, valueBuffer.getLong()); - } - else if (type.getJavaType() == double.class) { - preparedStatement.setDouble(parameterIndex, valueBuffer.getDouble()); - } - else if (type.getJavaType() == boolean.class) { - preparedStatement.setBoolean(parameterIndex, valueBuffer.getBoolean()); - } - else if (type.getJavaType() == Slice.class && isUuid) { - preparedStatement.setBytes(parameterIndex, valueBuffer.getSlice().getBytes()); - } - else if (type.getJavaType() == Slice.class) { - preparedStatement.setString(parameterIndex, new String(valueBuffer.getSlice().getBytes(), UTF_8)); - } - else { - throw new IllegalArgumentException("Unknown Java type: " + type.getJavaType()); - } - } - - private static int typeToSqlType(Type type) - { - if (type.equals(BIGINT)) { - return Types.BIGINT; - } - if (type.equals(DOUBLE)) { - return Types.DOUBLE; - } - if (type.equals(BOOLEAN)) { - return Types.BOOLEAN; - } - if (type instanceof VarcharType) { - return Types.VARCHAR; - } - if (type.equals(VARBINARY)) { - return Types.VARBINARY; - } - throw new IllegalArgumentException("Unknown type: " + type); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ResultSetValues.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ResultSetValues.java deleted file mode 100644 index 423d3c106f2a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ResultSetValues.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.base.VerifyException; -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.spi.type.Type; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; -import static io.airlift.slice.SizeOf.SIZE_OF_DOUBLE; -import static io.airlift.slice.SizeOf.SIZE_OF_LONG; -import static io.airlift.slice.Slices.wrappedBuffer; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static java.lang.String.format; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Locale.ENGLISH; -import static java.util.Objects.requireNonNull; - -public class ResultSetValues -{ - private final boolean[] booleans; - private final long[] longs; - private final double[] doubles; - private final String[] strings; - private final boolean[] nulls; - private final List types; - - public ResultSetValues(List types) - { - this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); - - this.booleans = new boolean[types.size()]; - this.longs = new long[types.size()]; - this.doubles = new double[types.size()]; - this.strings = new String[types.size()]; - this.nulls = new boolean[types.size()]; - } - - int extractValues(ResultSet resultSet, Set uuidColumns, Set hexColumns) - throws SQLException - { - checkArgument(resultSet != null, "resultSet is null"); - int completedBytes = 0; - - for (int i = 0; i < types.size(); i++) { - Class javaType = types.get(i).getJavaType(); - - if (javaType == boolean.class) { - booleans[i] = resultSet.getBoolean(i + 1); - nulls[i] = resultSet.wasNull(); - if (!nulls[i]) { - completedBytes += SIZE_OF_BYTE; - } - } - else if (javaType == long.class) { - longs[i] = resultSet.getLong(i + 1); - nulls[i] = resultSet.wasNull(); - if (!nulls[i]) { - completedBytes += SIZE_OF_LONG; - } - } - else if (javaType == double.class) { - doubles[i] = resultSet.getDouble(i + 1); - nulls[i] = resultSet.wasNull(); - if (!nulls[i]) { - completedBytes += SIZE_OF_DOUBLE; - } - } - else if (javaType == Slice.class) { - if (uuidColumns.contains(i)) { - byte[] bytes = resultSet.getBytes(i + 1); - nulls[i] = resultSet.wasNull(); - strings[i] = nulls[i] ? null : uuidFromBytes(bytes).toString().toLowerCase(ENGLISH); - } - else if (hexColumns.contains(i)) { - long value = resultSet.getLong(i + 1); - nulls[i] = resultSet.wasNull(); - strings[i] = nulls[i] ? null : format("%016x", value); - } - else { - String value = resultSet.getString(i + 1); - nulls[i] = resultSet.wasNull(); - strings[i] = nulls[i] ? null : value; - } - - if (!nulls[i]) { - completedBytes += strings[i].length(); - } - } - else { - throw new VerifyException("Unknown Java type: " + javaType); - } - } - return completedBytes; - } - - public boolean getBoolean(int field) - { - return booleans[field]; - } - - public long getLong(int field) - { - return longs[field]; - } - - public double getDouble(int field) - { - return doubles[field]; - } - - public Slice getSlice(int field) - { - return wrappedBuffer(strings[field].getBytes(UTF_8)); - } - - public boolean isNull(int field) - { - return nulls[field]; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataRecordCursor.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataRecordCursor.java deleted file mode 100644 index 51e4650f53e0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataRecordCursor.java +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.JdbiException; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Iterator; -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkPositionIndex; -import static com.google.common.base.Preconditions.checkState; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_CORRUPT_METADATA; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.maxColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.minColumn; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.metadataError; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static java.lang.String.format; -import static java.util.Collections.emptySet; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class ShardMetadataRecordCursor - implements RecordCursor -{ - private static final String SHARD_UUID = "shard_uuid"; - private static final String XXHASH64 = "xxhash64"; - private static final String SCHEMA_NAME = "table_schema"; - private static final String TABLE_NAME = "table_name"; - private static final String MIN_TIMESTAMP = "min_timestamp"; - private static final String MAX_TIMESTAMP = "max_timestamp"; - private static final String MIN_DATE = "min_date"; - private static final String MAX_DATE = "max_date"; - - public static final SchemaTableName SHARD_METADATA_TABLE_NAME = new SchemaTableName("system", "shards"); - public static final ConnectorTableMetadata SHARD_METADATA = new ConnectorTableMetadata( - SHARD_METADATA_TABLE_NAME, - ImmutableList.of( - new ColumnMetadata(SCHEMA_NAME, createUnboundedVarcharType()), - new ColumnMetadata(TABLE_NAME, createUnboundedVarcharType()), - new ColumnMetadata(SHARD_UUID, SHARD_UUID_COLUMN_TYPE), - new ColumnMetadata("bucket_number", BIGINT), - new ColumnMetadata("uncompressed_size", BIGINT), - new ColumnMetadata("compressed_size", BIGINT), - new ColumnMetadata("row_count", BIGINT), - new ColumnMetadata(XXHASH64, createVarcharType(16)), - new ColumnMetadata(MIN_TIMESTAMP, TIMESTAMP_MILLIS), - new ColumnMetadata(MAX_TIMESTAMP, TIMESTAMP_MILLIS), - new ColumnMetadata(MIN_DATE, DATE), - new ColumnMetadata(MAX_DATE, DATE))); - - private static final List COLUMNS = SHARD_METADATA.getColumns(); - private static final List TYPES = COLUMNS.stream().map(ColumnMetadata::getType).collect(toList()); - - private final Jdbi dbi; - private final MetadataDao metadataDao; - - private final Iterator tableIds; - private final List columnNames; - private final TupleDomain tupleDomain; - - private ResultSet resultSet; - private Connection connection; - private PreparedStatement statement; - private final ResultSetValues resultSetValues; - - private boolean closed; - private long completedBytes; - - public ShardMetadataRecordCursor(Jdbi dbi, TupleDomain tupleDomain) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - this.metadataDao = onDemandDao(dbi, MetadataDao.class); - this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); - this.tableIds = getTableIds(dbi, tupleDomain); - this.columnNames = createQualifiedColumnNames(); - this.resultSetValues = new ResultSetValues(TYPES); - this.resultSet = getNextResultSet(); - } - - private static String constructSqlTemplate(List columnNames, long tableId) - { - return format("SELECT %s\nFROM %s x\n" + - "JOIN shards ON (x.shard_id = shards.shard_id AND shards.table_id = %s)\n" + - "JOIN tables ON (tables.table_id = %s)\n", - Joiner.on(", ").join(columnNames), - shardIndexTable(tableId), - tableId, - tableId); - } - - private static List createQualifiedColumnNames() - { - return ImmutableList.builder() - .add("tables.schema_name") - .add("tables.table_name") - .add("shards." + COLUMNS.get(2).getName()) - .add("shards." + COLUMNS.get(3).getName()) - .add("shards." + COLUMNS.get(4).getName()) - .add("shards." + COLUMNS.get(5).getName()) - .add("shards." + COLUMNS.get(6).getName()) - .add("shards." + COLUMNS.get(7).getName()) - .add(MIN_TIMESTAMP) - .add(MAX_TIMESTAMP) - .add(MIN_DATE) - .add(MAX_DATE) - .build(); - } - - @Override - public long getCompletedBytes() - { - return completedBytes; - } - - @Override - public long getReadTimeNanos() - { - return 0; - } - - @Override - public Type getType(int field) - { - checkPositionIndex(field, TYPES.size()); - return TYPES.get(field); - } - - @Override - public boolean advanceNextPosition() - { - if (resultSet == null) { - close(); - } - - if (closed) { - return false; - } - - try { - while (!resultSet.next()) { - resultSet = getNextResultSet(); - if (resultSet == null) { - close(); - return false; - } - } - completedBytes += resultSetValues.extractValues( - resultSet, - ImmutableSet.of(getColumnIndex(SHARD_METADATA, SHARD_UUID)), - ImmutableSet.of(getColumnIndex(SHARD_METADATA, XXHASH64))); - return true; - } - catch (SQLException | JdbiException e) { - throw metadataError(e); - } - } - - @Override - public boolean getBoolean(int field) - { - checkFieldType(field, boolean.class); - return resultSetValues.getBoolean(field); - } - - @Override - public long getLong(int field) - { - checkFieldType(field, long.class); - return resultSetValues.getLong(field); - } - - @Override - public double getDouble(int field) - { - checkFieldType(field, double.class); - return resultSetValues.getDouble(field); - } - - @Override - public Slice getSlice(int field) - { - checkFieldType(field, Slice.class); - return resultSetValues.getSlice(field); - } - - @Override - public Object getObject(int field) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isNull(int field) - { - checkState(!closed, "cursor is closed"); - checkPositionIndex(field, TYPES.size()); - return resultSetValues.isNull(field); - } - - @Override - public void close() - { - closed = true; - closeCurrentResultSet(); - } - - @SuppressWarnings("unused") - private void closeCurrentResultSet() - { - // use try-with-resources to close everything properly - //noinspection EmptyTryBlock - try (Connection connection = this.connection; - Statement statement = this.statement; - ResultSet resultSet = this.resultSet) { - // do nothing - } - catch (SQLException _) { - } - } - - private ResultSet getNextResultSet() - { - closeCurrentResultSet(); - - if (!tableIds.hasNext()) { - return null; - } - - Long tableId = tableIds.next(); - Long columnId = metadataDao.getTemporalColumnId(tableId); - List columnNames; - - if (columnId == null) { - columnNames = getMappedColumnNames("null", "null", "null", "null"); - } - else { - Type temporalType = metadataDao.getTableColumn(tableId, columnId).getDataType(); - if (temporalType.equals(DATE)) { - columnNames = getMappedColumnNames("null", "null", minColumn(columnId), maxColumn(columnId)); - } - else if (temporalType.equals(TIMESTAMP_MILLIS)) { - columnNames = getMappedColumnNames(minColumn(columnId), maxColumn(columnId), "null", "null"); - } - else { - throw new TrinoException(RAPTOR_CORRUPT_METADATA, "Temporal column should be of type date or timestamp, not " + temporalType.getDisplayName()); - } - } - - try { - connection = dbi.open().getConnection(); - statement = PreparedStatementBuilder.create( - connection, - constructSqlTemplate(columnNames, tableId), - columnNames, - TYPES, - ImmutableSet.of(getColumnIndex(SHARD_METADATA, SHARD_UUID)), - tupleDomain); - return statement.executeQuery(); - } - catch (SQLException | JdbiException e) { - close(); - throw metadataError(e); - } - } - - private List getMappedColumnNames(String minTimestampColumn, String maxTimestampColumn, String minDateColumn, String maxDateColumn) - { - ImmutableList.Builder builder = ImmutableList.builder(); - for (String column : columnNames) { - switch (column) { - case MIN_TIMESTAMP: - builder.add(minTimestampColumn); - break; - case MAX_TIMESTAMP: - builder.add(maxTimestampColumn); - break; - case MIN_DATE: - builder.add(minDateColumn); - break; - case MAX_DATE: - builder.add(maxDateColumn); - break; - default: - builder.add(column); - break; - } - } - return builder.build(); - } - - @VisibleForTesting - static Iterator getTableIds(Jdbi dbi, TupleDomain tupleDomain) - { - ImmutableList.Builder tableIds = ImmutableList.builder(); - try (Connection connection = dbi.open().getConnection(); - PreparedStatement statement = PreparedStatementBuilder.create( - connection, - "SELECT table_id FROM tables ", - List.of("schema_name", "table_name"), - List.of(VARCHAR, VARCHAR), - emptySet(), - tupleDomain.filter((key, domain) -> (key == 0) || (key == 1))); - ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - tableIds.add(resultSet.getLong("table_id")); - } - } - catch (SQLException | JdbiException e) { - throw metadataError(e); - } - return tableIds.build().iterator(); - } - - private static int getColumnIndex(ConnectorTableMetadata tableMetadata, String columnName) - { - List columns = tableMetadata.getColumns(); - for (int i = 0; i < columns.size(); i++) { - if (columns.get(i).getName().equals(columnName)) { - return i; - } - } - throw new IllegalArgumentException(format("Column '%s' not found", columnName)); - } - - private void checkFieldType(int field, Class clazz) - { - checkState(!closed, "cursor is closed"); - Type type = getType(field); - checkArgument(type.getJavaType() == clazz, "Type %s cannot be read as %s", type, clazz.getSimpleName()); - } - - private static String getStringValue(Object value) - { - return ((Slice) value).toStringUtf8(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataSystemTable.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataSystemTable.java deleted file mode 100644 index b4e7a8364746..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ShardMetadataSystemTable.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.predicate.TupleDomain; -import org.jdbi.v3.core.Jdbi; - -import static io.trino.plugin.raptor.legacy.systemtables.ShardMetadataRecordCursor.SHARD_METADATA; -import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; -import static java.util.Objects.requireNonNull; - -public class ShardMetadataSystemTable - implements SystemTable -{ - private final Jdbi dbi; - - @Inject - public ShardMetadataSystemTable(@ForMetadata Jdbi dbi) - { - this.dbi = requireNonNull(dbi, "dbi is null"); - } - - @Override - public Distribution getDistribution() - { - return SINGLE_COORDINATOR; - } - - @Override - public ConnectorTableMetadata getTableMetadata() - { - return SHARD_METADATA; - } - - @Override - public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) - { - return new ShardMetadataRecordCursor(dbi, constraint); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java deleted file mode 100644 index 28192417bca6..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.PeekingIterator; -import com.google.inject.Inject; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.metadata.ColumnMetadataRow; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.TableMetadataRow; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.block.ArrayBlockBuilder; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.FixedPageSource; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.predicate.NullableValue; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.TypeManager; -import org.jdbi.v3.core.Jdbi; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.SortedMap; -import java.util.TreeMap; - -import static com.google.common.collect.Iterators.peekingIterator; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_CORRUPT_METADATA; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; -import static io.trino.spi.predicate.TupleDomain.extractFixedValues; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public class TableMetadataSystemTable - implements SystemTable -{ - private static final String TABLE_NAME = "table_name"; - private static final String SCHEMA_NAME = "table_schema"; - - private final MetadataDao dao; - private final ConnectorTableMetadata tableMetadata; - - @Inject - public TableMetadataSystemTable(@ForMetadata Jdbi dbi, TypeManager typeManager) - { - this.dao = onDemandDao(dbi, MetadataDao.class); - requireNonNull(typeManager, "typeManager is null"); - - this.tableMetadata = new ConnectorTableMetadata( - new SchemaTableName("system", "tables"), - ImmutableList.of( - new ColumnMetadata(SCHEMA_NAME, VARCHAR), - new ColumnMetadata(TABLE_NAME, VARCHAR), - new ColumnMetadata("temporal_column", VARCHAR), - new ColumnMetadata("ordering_columns", new ArrayType(VARCHAR)), - new ColumnMetadata("distribution_name", VARCHAR), - new ColumnMetadata("bucket_count", BIGINT), - new ColumnMetadata("bucketing_columns", new ArrayType(VARCHAR)), - new ColumnMetadata("organized", BOOLEAN))); - } - - @Override - public Distribution getDistribution() - { - return SINGLE_COORDINATOR; - } - - @Override - public ConnectorTableMetadata getTableMetadata() - { - return tableMetadata; - } - - @Override - public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) - { - return new FixedPageSource(buildPages(dao, tableMetadata, constraint)); - } - - private static List buildPages(MetadataDao dao, ConnectorTableMetadata tableMetadata, TupleDomain tupleDomain) - { - Map domainValues = extractFixedValues(tupleDomain).orElse(ImmutableMap.of()); - String schemaName = getStringValue(domainValues.get(getColumnIndex(tableMetadata, SCHEMA_NAME))); - String tableName = getStringValue(domainValues.get(getColumnIndex(tableMetadata, TABLE_NAME))); - - PageListBuilder pageBuilder = new PageListBuilder(tableMetadata.getColumns().stream() - .map(ColumnMetadata::getType) - .collect(toList())); - - List tableRows = dao.getTableMetadataRows(schemaName, tableName); - PeekingIterator columnRowIterator = peekingIterator(dao.getColumnMetadataRows(schemaName, tableName).iterator()); - - for (TableMetadataRow tableRow : tableRows) { - while (columnRowIterator.hasNext() && columnRowIterator.peek().getTableId() < tableRow.getTableId()) { - columnRowIterator.next(); - } - - String temporalColumnName = null; - SortedMap sortColumnNames = new TreeMap<>(); - SortedMap bucketColumnNames = new TreeMap<>(); - OptionalLong temporalColumnId = tableRow.getTemporalColumnId(); - while (columnRowIterator.hasNext() && columnRowIterator.peek().getTableId() == tableRow.getTableId()) { - ColumnMetadataRow columnRow = columnRowIterator.next(); - if (temporalColumnId.isPresent() && columnRow.getColumnId() == temporalColumnId.getAsLong()) { - temporalColumnName = columnRow.getColumnName(); - } - OptionalInt sortOrdinalPosition = columnRow.getSortOrdinalPosition(); - if (sortOrdinalPosition.isPresent()) { - sortColumnNames.put(sortOrdinalPosition.getAsInt(), columnRow.getColumnName()); - } - OptionalInt bucketOrdinalPosition = columnRow.getBucketOrdinalPosition(); - if (bucketOrdinalPosition.isPresent()) { - bucketColumnNames.put(bucketOrdinalPosition.getAsInt(), columnRow.getColumnName()); - } - } - - pageBuilder.beginRow(); - - // schema_name, table_name - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(tableRow.getSchemaName())); - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(tableRow.getTableName())); - - // temporal_column - if (temporalColumnId.isPresent()) { - if (temporalColumnName == null) { - throw new TrinoException(RAPTOR_CORRUPT_METADATA, format("Table ID %s has corrupt metadata (invalid temporal column ID)", tableRow.getTableId())); - } - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(temporalColumnName)); - } - else { - pageBuilder.nextBlockBuilder().appendNull(); - } - - // ordering_columns - writeArray(pageBuilder.nextBlockBuilder(), sortColumnNames.values()); - - // distribution_name - Optional distributionName = tableRow.getDistributionName(); - if (distributionName.isPresent()) { - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(distributionName.get())); - } - else { - pageBuilder.nextBlockBuilder().appendNull(); - } - - // bucket_count - OptionalInt bucketCount = tableRow.getBucketCount(); - if (bucketCount.isPresent()) { - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), bucketCount.getAsInt()); - } - else { - pageBuilder.nextBlockBuilder().appendNull(); - } - - // bucketing_columns - writeArray(pageBuilder.nextBlockBuilder(), bucketColumnNames.values()); - - // organized - BOOLEAN.writeBoolean(pageBuilder.nextBlockBuilder(), tableRow.isOrganized()); - } - - return pageBuilder.build(); - } - - private static void writeArray(BlockBuilder blockBuilder, Collection values) - { - if (values.isEmpty()) { - blockBuilder.appendNull(); - } - else { - ((ArrayBlockBuilder) blockBuilder).buildEntry(elementBuilder -> { - for (String value : values) { - VARCHAR.writeSlice(elementBuilder, utf8Slice(value)); - } - }); - } - } - - static int getColumnIndex(ConnectorTableMetadata tableMetadata, String columnName) - { - List columns = tableMetadata.getColumns(); - for (int i = 0; i < columns.size(); i++) { - if (columns.get(i).getName().equals(columnName)) { - return i; - } - } - throw new IllegalArgumentException(format("Column '%s' not found", columnName)); - } - - static String getStringValue(NullableValue value) - { - if ((value == null) || value.isNull()) { - return null; - } - return ((Slice) value.getValue()).toStringUtf8(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableStatsSystemTable.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableStatsSystemTable.java deleted file mode 100644 index bde918a3781b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/TableStatsSystemTable.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; -import io.trino.plugin.raptor.legacy.metadata.ForMetadata; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.TableStatsRow; -import io.trino.spi.Page; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.FixedPageSource; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SystemTable; -import io.trino.spi.predicate.NullableValue; -import io.trino.spi.predicate.TupleDomain; -import org.jdbi.v3.core.Jdbi; - -import java.util.List; -import java.util.Map; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.systemtables.TableMetadataSystemTable.getColumnIndex; -import static io.trino.plugin.raptor.legacy.systemtables.TableMetadataSystemTable.getStringValue; -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; -import static io.trino.spi.predicate.TupleDomain.extractFixedValues; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static java.util.stream.Collectors.toList; - -public class TableStatsSystemTable - implements SystemTable -{ - private static final String SCHEMA_NAME = "table_schema"; - private static final String TABLE_NAME = "table_name"; - - private static final ConnectorTableMetadata METADATA = new ConnectorTableMetadata( - new SchemaTableName("system", "table_stats"), - ImmutableList.builder() - .add(new ColumnMetadata(SCHEMA_NAME, createUnboundedVarcharType())) - .add(new ColumnMetadata(TABLE_NAME, createUnboundedVarcharType())) - .add(new ColumnMetadata("create_time", TIMESTAMP_MILLIS)) - .add(new ColumnMetadata("update_time", TIMESTAMP_MILLIS)) - .add(new ColumnMetadata("table_version", BIGINT)) - .add(new ColumnMetadata("shard_count", BIGINT)) - .add(new ColumnMetadata("row_count", BIGINT)) - .add(new ColumnMetadata("compressed_size", BIGINT)) - .add(new ColumnMetadata("uncompressed_size", BIGINT)) - .build()); - - private final MetadataDao dao; - - @Inject - public TableStatsSystemTable(@ForMetadata Jdbi dbi) - { - this.dao = onDemandDao(dbi, MetadataDao.class); - } - - @Override - public Distribution getDistribution() - { - return SINGLE_COORDINATOR; - } - - @Override - public ConnectorTableMetadata getTableMetadata() - { - return METADATA; - } - - @Override - public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) - { - return new FixedPageSource(buildPages(dao, constraint)); - } - - private static List buildPages(MetadataDao dao, TupleDomain tupleDomain) - { - Map domainValues = extractFixedValues(tupleDomain).orElse(ImmutableMap.of()); - String schemaName = getStringValue(domainValues.get(getColumnIndex(METADATA, SCHEMA_NAME))); - String tableName = getStringValue(domainValues.get(getColumnIndex(METADATA, TABLE_NAME))); - - PageListBuilder pageBuilder = new PageListBuilder(METADATA.getColumns().stream() - .map(ColumnMetadata::getType) - .collect(toList())); - - for (TableStatsRow row : dao.getTableStatsRows(schemaName, tableName)) { - pageBuilder.beginRow(); - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(row.getSchemaName())); - VARCHAR.writeSlice(pageBuilder.nextBlockBuilder(), utf8Slice(row.getTableName())); - TIMESTAMP_MILLIS.writeLong(pageBuilder.nextBlockBuilder(), row.getCreateTime() * MICROSECONDS_PER_MILLISECOND); - TIMESTAMP_MILLIS.writeLong(pageBuilder.nextBlockBuilder(), row.getUpdateTime() * MICROSECONDS_PER_MILLISECOND); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getTableVersion()); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getShardCount()); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getRowCount()); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getCompressedSize()); - BIGINT.writeLong(pageBuilder.nextBlockBuilder(), row.getUncompressedSize()); - } - - return pageBuilder.build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ValueBuffer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ValueBuffer.java deleted file mode 100644 index 97f7bdf56c9b..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/systemtables/ValueBuffer.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.primitives.Primitives; -import io.airlift.slice.Slice; -import io.trino.spi.type.Type; -import jakarta.annotation.Nullable; - -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.slice.Slices.wrappedBuffer; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public class ValueBuffer -{ - private final int columnIndex; - private final Type type; - private final Object value; - - private ValueBuffer(int columnIndex, Type type, @Nullable Object value) - { - checkArgument(columnIndex >= 0, "columnIndex must be > 0"); - this.columnIndex = columnIndex; - this.type = requireNonNull(type, "type is null"); - this.value = value; - } - - public static ValueBuffer create(int columnIndex, Type type, @Nullable Object value) - { - if (value == null) { - return new ValueBuffer(columnIndex, type, null); - } - return new ValueBuffer(columnIndex, type, normalizeValue(type, value)); - } - - private static Object normalizeValue(Type type, @Nullable Object value) - { - if (value == null) { - return value; - } - - Class javaType = type.getJavaType(); - if (javaType.isPrimitive()) { - checkArgument(Primitives.wrap(javaType).isInstance(value), "Type %s incompatible with %s", type, value); - return value; - } - if (javaType == Slice.class) { - if (value instanceof Slice) { - return value; - } - if (value instanceof String) { - return utf8Slice(((String) value)); - } - if (value instanceof byte[]) { - return wrappedBuffer((byte[]) value); - } - } - throw new IllegalArgumentException(format("Type %s incompatible with %s", type, value)); - } - - @Nullable - public Object getValue() - { - checkState(!isNull()); - return value; - } - - public int getColumnIndex() - { - return columnIndex; - } - - public long getLong() - { - checkState(!isNull()); - checkArgument(type.getJavaType() == long.class, "Type %s cannot be read as long", type); - return (Long) value; - } - - public double getDouble() - { - checkState(!isNull()); - checkArgument(type.getJavaType() == double.class, "Type %s cannot be read as double", type); - return (Double) value; - } - - public boolean getBoolean() - { - checkState(!isNull()); - checkArgument(type.getJavaType() == boolean.class, "Type %s cannot be read as boolean", type); - return (Boolean) value; - } - - public Slice getSlice() - { - checkState(!isNull()); - checkArgument(type.getJavaType() == Slice.class, "Type %s cannot be read as Slice", type); - return (Slice) value; - } - - public Type getType() - { - return type; - } - - public boolean isNull() - { - return value == null; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ValueBuffer that = (ValueBuffer) o; - return columnIndex == that.columnIndex && - Objects.equals(type, that.type) && - Objects.equals(value, that.value); - } - - @Override - public int hashCode() - { - return Objects.hash(columnIndex, type, value); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("columnIndex", columnIndex) - .add("type", type) - .add("value", value) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ArrayUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ArrayUtil.java deleted file mode 100644 index c5c1ad35b0e0..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ArrayUtil.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.collect.ImmutableList; - -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.List; - -public final class ArrayUtil -{ - private ArrayUtil() {} - - /** - * Unpack an array of big endian ints. - */ - public static List intArrayFromBytes(byte[] bytes) - { - ImmutableList.Builder list = ImmutableList.builder(); - ByteBuffer buffer = ByteBuffer.wrap(bytes); - while (buffer.hasRemaining()) { - list.add(buffer.getInt()); - } - return list.build(); - } - - /** - * Pack an array of ints as big endian. - */ - public static byte[] intArrayToBytes(Collection values) - { - ByteBuffer buffer = ByteBuffer.allocate(values.size() * Integer.BYTES); - for (int value : values) { - buffer.putInt(value); - } - return buffer.array(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/Closer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/Closer.java deleted file mode 100644 index 4a79a1a144d8..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/Closer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import static java.util.Objects.requireNonNull; - -public class Closer - implements AutoCloseable -{ - private final T delegate; - private final Cleaner cleaner; - - public static Closer closer(T delegate, Cleaner cleaner) - { - return new Closer<>(delegate, cleaner); - } - - private Closer(T delegate, Cleaner cleaner) - { - this.delegate = requireNonNull(delegate, "delegate is null"); - this.cleaner = requireNonNull(cleaner, "cleaner is null"); - } - - public T get() - { - return delegate; - } - - @Override - public void close() - throws X - { - cleaner.close(delegate); - } - - public interface Cleaner - { - void close(T object) - throws X; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java deleted file mode 100644 index 3d346df637ef..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/ConcatPageSource.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import io.trino.spi.Page; -import io.trino.spi.connector.ConnectorPageSource; - -import java.io.IOException; -import java.util.Iterator; - -import static java.util.Objects.requireNonNull; - -public class ConcatPageSource - implements ConnectorPageSource -{ - private final Iterator iterator; - - private ConnectorPageSource current; - private long completedBytes; - private long readTimeNanos; - - public ConcatPageSource(Iterator iterator) - { - this.iterator = requireNonNull(iterator, "iterator is null"); - } - - @Override - public long getCompletedBytes() - { - setup(); - return completedBytes + ((current != null) ? current.getCompletedBytes() : 0); - } - - @Override - public long getReadTimeNanos() - { - setup(); - return readTimeNanos + ((current != null) ? current.getReadTimeNanos() : 0); - } - - @Override - public boolean isFinished() - { - setup(); - return current == null; - } - - @Override - public Page getNextPage() - { - while (true) { - setup(); - - if (current == null) { - return null; - } - if (!current.isFinished()) { - return current.getNextPage(); - } - - completedBytes += current.getCompletedBytes(); - readTimeNanos += current.getReadTimeNanos(); - current = null; - } - } - - @Override - public long getMemoryUsage() - { - return (current != null) ? current.getMemoryUsage() : 0; - } - - @Override - public void close() - throws IOException - { - if (current != null) { - current.close(); - } - } - - private void setup() - { - if ((current == null) && iterator.hasNext()) { - current = iterator.next(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DaoSupplier.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DaoSupplier.java deleted file mode 100644 index 7bfed77b2685..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DaoSupplier.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; - -import static io.trino.plugin.raptor.legacy.util.DatabaseUtil.onDemandDao; -import static java.util.Objects.requireNonNull; - -public class DaoSupplier -{ - private final Class type; - private final T dao; - - public DaoSupplier(Jdbi dbi, Class type) - { - requireNonNull(dbi, "dbi is null"); - requireNonNull(type, "type is null"); - this.type = type; - this.dao = onDemandDao(dbi, type); - } - - public T onDemand() - { - return dao; - } - - public T attach(Handle handle) - { - return handle.attach(type); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DatabaseUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DatabaseUtil.java deleted file mode 100644 index 3e6604bfb289..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/DatabaseUtil.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.base.Throwables; -import com.mysql.cj.jdbc.JdbcStatement; -import io.trino.spi.TrinoException; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.HandleCallback; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.JdbiException; - -import java.lang.reflect.InvocationTargetException; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.function.Consumer; -import java.util.function.Predicate; - -import static com.google.common.base.Throwables.throwIfInstanceOf; -import static com.google.common.reflect.Reflection.newProxy; -import static com.mysql.cj.exceptions.MysqlErrorNumbers.ER_TRANS_CACHE_FULL; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_METADATA_ERROR; -import static java.sql.Types.INTEGER; -import static java.util.Objects.requireNonNull; - -public final class DatabaseUtil -{ - private DatabaseUtil() {} - - public static T onDemandDao(Jdbi dbi, Class daoType) - { - requireNonNull(dbi, "dbi is null"); - return newProxy(daoType, (proxy, method, args) -> { - try (Handle handle = dbi.open()) { - T dao = handle.attach(daoType); - return method.invoke(dao, args); - } - catch (JdbiException e) { - throw metadataError(e); - } - catch (InvocationTargetException e) { - throw metadataError(e.getCause()); - } - }); - } - - public static T runTransaction(Jdbi dbi, HandleCallback callback) - { - try { - return dbi.inTransaction(callback); - } - catch (JdbiException e) { - if (e.getCause() != null) { - throwIfInstanceOf(e.getCause(), TrinoException.class); - } - throw metadataError(e); - } - } - - public static void daoTransaction(Jdbi dbi, Class daoType, Consumer callback) - { - runTransaction(dbi, handle -> { - callback.accept(handle.attach(daoType)); - return null; - }); - } - - public static TrinoException metadataError(Throwable cause, String message) - { - return new TrinoException(RAPTOR_METADATA_ERROR, message, cause); - } - - public static TrinoException metadataError(Throwable cause) - { - return metadataError(cause, "Failed to perform metadata operation"); - } - - /** - * Run an SQL query as ignoring any constraint violations. - * This allows idempotent inserts (equivalent to INSERT IGNORE). - */ - public static void runIgnoringConstraintViolation(Runnable task) - { - try { - task.run(); - } - catch (RuntimeException e) { - if (!sqlCodeStartsWith(e, "23")) { - throw e; - } - } - } - - public static void enableStreamingResults(Statement statement) - throws SQLException - { - if (statement.isWrapperFor(JdbcStatement.class)) { - statement.unwrap(JdbcStatement.class).enableStreamingResults(); - } - } - - public static OptionalInt getOptionalInt(ResultSet rs, String name) - throws SQLException - { - int value = rs.getInt(name); - return rs.wasNull() ? OptionalInt.empty() : OptionalInt.of(value); - } - - public static OptionalLong getOptionalLong(ResultSet rs, String name) - throws SQLException - { - long value = rs.getLong(name); - return rs.wasNull() ? OptionalLong.empty() : OptionalLong.of(value); - } - - public static Long getBoxedLong(ResultSet rs, String name) - throws SQLException - { - long value = rs.getLong(name); - return rs.wasNull() ? null : value; - } - - public static void bindOptionalInt(PreparedStatement statement, int index, OptionalInt value) - throws SQLException - { - if (value.isPresent()) { - statement.setInt(index, value.getAsInt()); - } - else { - statement.setNull(index, INTEGER); - } - } - - public static boolean isSyntaxOrAccessError(Exception e) - { - return sqlCodeStartsWith(e, "42"); - } - - public static boolean isTransactionCacheFullError(Exception e) - { - return mySqlErrorCodeMatches(e, ER_TRANS_CACHE_FULL); - } - - /** - * Check if an exception is caused by a MySQL exception of certain error code - */ - private static boolean mySqlErrorCodeMatches(Exception e, int errorCode) - { - return Throwables.getCausalChain(e).stream() - .filter(SQLException.class::isInstance) - .map(SQLException.class::cast) - .filter(t -> t.getErrorCode() == errorCode) - .map(Throwable::getStackTrace) - .anyMatch(isMySQLException()); - } - - private static Predicate isMySQLException() - { - // check if the exception is a mysql exception by matching the package name in the stack trace - return s -> Arrays.stream(s) - .map(StackTraceElement::getClassName) - .anyMatch(t -> t.startsWith("com.mysql.jdbc.")); - } - - private static boolean sqlCodeStartsWith(Exception e, String code) - { - for (Throwable throwable : Throwables.getCausalChain(e)) { - if (throwable instanceof SQLException) { - String state = ((SQLException) throwable).getSQLState(); - if (state != null && state.startsWith(code)) { - return true; - } - } - } - return false; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/MetadataUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/MetadataUtil.java deleted file mode 100644 index 96262838bf5a..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/MetadataUtil.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.util.Locale.ENGLISH; - -public final class MetadataUtil -{ - private MetadataUtil() {} - - public static String checkSchemaName(String schemaName) - { - return checkLowerCase(schemaName, "schemaName"); - } - - public static String checkTableName(String tableName) - { - return checkLowerCase(tableName, "tableName"); - } - - private static String checkLowerCase(String value, String name) - { - if (value == null) { - throw new NullPointerException(format("%s is null", name)); - } - checkArgument(value.equals(value.toLowerCase(ENGLISH)), "%s is not lowercase", name); - return value; - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PageBuffer.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PageBuffer.java deleted file mode 100644 index 5a9d60e3872d..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PageBuffer.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.collect.ImmutableList; -import io.trino.plugin.raptor.legacy.storage.StoragePageSink; -import io.trino.spi.Page; -import io.trino.spi.PageSorter; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.type.Type; - -import java.util.ArrayList; -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.Math.toIntExact; -import static java.util.Objects.requireNonNull; - -public class PageBuffer -{ - private final long maxMemoryBytes; - private final StoragePageSink storagePageSink; - private final List columnTypes; - private final List sortFields; - private final List sortOrders; - private final PageSorter pageSorter; - private final List pages = new ArrayList<>(); - - private long usedMemoryBytes; - private long rowCount; - - public PageBuffer( - long maxMemoryBytes, - StoragePageSink storagePageSink, - List columnTypes, - List sortFields, - List sortOrders, - PageSorter pageSorter) - { - checkArgument(maxMemoryBytes > 0, "maxMemoryBytes must be positive"); - this.maxMemoryBytes = maxMemoryBytes; - this.columnTypes = requireNonNull(columnTypes, "columnTypes is null"); - this.sortFields = ImmutableList.copyOf(requireNonNull(sortFields, "sortFields is null")); - this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); - this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); - this.storagePageSink = requireNonNull(storagePageSink, "storagePageSink is null"); - } - - public StoragePageSink getStoragePageSink() - { - return storagePageSink; - } - - public long getUsedMemoryBytes() - { - return usedMemoryBytes; - } - - public void add(Page page) - { - flushIfNecessary(page.getPositionCount()); - pages.add(page); - usedMemoryBytes += page.getSizeInBytes(); - rowCount += page.getPositionCount(); - } - - public void flush() - { - if (pages.isEmpty()) { - return; - } - - if (sortFields.isEmpty()) { - storagePageSink.appendPages(pages); - } - else { - appendSorted(); - } - - storagePageSink.flush(); - - pages.clear(); - rowCount = 0; - usedMemoryBytes = 0; - } - - private void appendSorted() - { - long[] addresses = pageSorter.sort(columnTypes, pages, sortFields, sortOrders, toIntExact(rowCount)); - - int[] pageIndex = new int[addresses.length]; - int[] positionIndex = new int[addresses.length]; - for (int i = 0; i < addresses.length; i++) { - pageIndex[i] = pageSorter.decodePageIndex(addresses[i]); - positionIndex[i] = pageSorter.decodePositionIndex(addresses[i]); - } - - storagePageSink.appendPages(pages, pageIndex, positionIndex); - } - - private void flushIfNecessary(int rowsToAdd) - { - if (storagePageSink.isFull() || !canAddRows(rowsToAdd)) { - flush(); - } - } - - private boolean canAddRows(int rowsToAdd) - { - return (usedMemoryBytes < maxMemoryBytes) && - ((rowCount + rowsToAdd) < Integer.MAX_VALUE); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PrioritizedFifoExecutor.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PrioritizedFifoExecutor.java deleted file mode 100644 index a4094be27522..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/PrioritizedFifoExecutor.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.collect.ComparisonChain; -import com.google.common.util.concurrent.ExecutionList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.errorprone.annotations.ThreadSafe; -import io.airlift.log.Logger; - -import java.util.Comparator; -import java.util.Objects; -import java.util.Queue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.FutureTask; -import java.util.concurrent.PriorityBlockingQueue; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -/** - * This class is based on io.airlift.concurrent.BoundedExecutor - */ -@ThreadSafe -public class PrioritizedFifoExecutor -{ - private static final Logger log = Logger.get(PrioritizedFifoExecutor.class); - - private final Queue> queue; - private final AtomicInteger queueSize = new AtomicInteger(0); - private final AtomicLong sequenceNumber = new AtomicLong(0); - private final Runnable triggerTask = this::executeOrMerge; - - private final ExecutorService executorService; - private final int maxThreads; - private final Comparator taskComparator; - - public PrioritizedFifoExecutor(ExecutorService coreExecutor, int maxThreads, Comparator taskComparator) - { - checkArgument(maxThreads > 0, "maxThreads must be greater than zero"); - - this.taskComparator = requireNonNull(taskComparator, "taskComparator is null"); - this.executorService = requireNonNull(coreExecutor, "coreExecutor is null"); - this.maxThreads = maxThreads; - this.queue = new PriorityBlockingQueue<>(maxThreads); - } - - public ListenableFuture submit(T task) - { - FifoRunnableTask fifoTask = new FifoRunnableTask<>(task, sequenceNumber.incrementAndGet(), taskComparator); - queue.add(fifoTask); - executorService.submit(triggerTask); - return fifoTask; - } - - private void executeOrMerge() - { - int size = queueSize.incrementAndGet(); - if (size > maxThreads) { - return; - } - do { - try { - queue.poll().run(); - } - catch (Throwable e) { - log.error(e, "Task failed"); - } - } - while (queueSize.getAndDecrement() > maxThreads); - } - - private static class FifoRunnableTask - extends FutureTask - implements ListenableFuture, Comparable> - { - private final ExecutionList executionList = new ExecutionList(); - private final T task; - private final long sequenceNumber; - private final Comparator taskComparator; - - public FifoRunnableTask(T task, long sequenceNumber, Comparator taskComparator) - { - super(requireNonNull(task, "task is null"), null); - this.task = task; - this.sequenceNumber = sequenceNumber; - this.taskComparator = requireNonNull(taskComparator, "taskComparator is null"); - } - - @Override - public void addListener(Runnable listener, Executor executor) - { - executionList.add(listener, executor); - } - - @Override - protected void done() - { - executionList.execute(); - } - - @Override - public int compareTo(FifoRunnableTask other) - { - return ComparisonChain.start() - .compare(this.task, other.task, taskComparator) - .compare(this.sequenceNumber, other.sequenceNumber) - .result(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - FifoRunnableTask other = (FifoRunnableTask) o; - return Objects.equals(this.task, other.task) && - this.sequenceNumber == other.sequenceNumber; - } - - @Override - public int hashCode() - { - return Objects.hash(task, sequenceNumber); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SynchronizedResultIterator.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SynchronizedResultIterator.java deleted file mode 100644 index 2562cb727761..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SynchronizedResultIterator.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.errorprone.annotations.concurrent.GuardedBy; -import org.jdbi.v3.core.result.ResultIterator; -import org.jdbi.v3.core.statement.StatementContext; - -import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; - -public class SynchronizedResultIterator - implements ResultIterator -{ - @GuardedBy("this") - private final ResultIterator iterator; - - @GuardedBy("this") - private boolean closed; - - public SynchronizedResultIterator(ResultIterator iterator) - { - this.iterator = requireNonNull(iterator, "iterator is null"); - } - - @Override - public synchronized boolean hasNext() - { - checkState(!closed, "already closed"); - return iterator.hasNext(); - } - - @Override - public synchronized T next() - { - checkState(!closed, "already closed"); - return iterator.next(); - } - - @Override - public synchronized void close() - { - if (!closed) { - closed = true; - iterator.close(); - } - } - - @Override - public synchronized StatementContext getContext() - { - return iterator.getContext(); - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SyncingFileOutputStream.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SyncingFileOutputStream.java deleted file mode 100644 index c952b0bb682e..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/SyncingFileOutputStream.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import com.google.common.io.Closer; -import io.airlift.slice.XxHash64; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; - -public class SyncingFileOutputStream - extends FileOutputStream -{ - private final byte[] oneByte = new byte[1]; - private final XxHash64 hash = new XxHash64(); - private final File file; - private boolean closed; - - public SyncingFileOutputStream(File file) - throws FileNotFoundException - { - super(file); - this.file = file; - } - - @Override - public void write(byte[] b, int off, int len) - throws IOException - { - super.write(b, off, len); - hash.update(b, off, len); - } - - @SuppressWarnings("NumericCastThatLosesPrecision") - @Override - public void write(int b) - throws IOException - { - oneByte[0] = (byte) b; - write(oneByte, 0, 1); - } - - @Override - public void close() - throws IOException - { - if (closed) { - return; - } - closed = true; - - try (Closer closer = Closer.create()) { - closer.register(super::close); - closer.register(() -> getFD().sync()); - closer.register(this::flush); - } - - // extremely paranoid code to detect a broken local file system - try (InputStream in = new FileInputStream(file)) { - if (hash.hash() != XxHash64.hash(in)) { - throw new IOException("File is corrupt after write"); - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/UuidUtil.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/UuidUtil.java deleted file mode 100644 index 724d52c2c4c9..000000000000 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/util/UuidUtil.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import org.jdbi.v3.core.argument.AbstractArgumentFactory; -import org.jdbi.v3.core.argument.Argument; -import org.jdbi.v3.core.argument.internal.strategies.LoggableBinderArgument; -import org.jdbi.v3.core.config.ConfigRegistry; -import org.jdbi.v3.core.mapper.ColumnMapper; -import org.jdbi.v3.core.statement.StatementContext; - -import java.nio.ByteBuffer; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Types; -import java.util.UUID; - -public final class UuidUtil -{ - private UuidUtil() {} - - public static final class UuidArgumentFactory - extends AbstractArgumentFactory - { - public UuidArgumentFactory() - { - super(Types.VARBINARY); - } - - @Override - protected Argument build(UUID uuid, ConfigRegistry config) - { - return new LoggableBinderArgument<>(uuid, (statement, index, value) -> - statement.setBytes(index, uuidToBytes(value))); - } - } - - public static final class UuidColumnMapper - implements ColumnMapper - { - @Override - public UUID map(ResultSet r, int index, StatementContext ctx) - throws SQLException - { - return uuidFromBytes(r.getBytes(index)); - } - } - - public static UUID uuidFromBytes(byte[] bytes) - { - ByteBuffer buffer = ByteBuffer.wrap(bytes); - long msb = buffer.getLong(); - long lsb = buffer.getLong(); - return new UUID(msb, lsb); - } - - public static byte[] uuidToBytes(UUID uuid) - { - return ByteBuffer.allocate(16) - .putLong(uuid.getMostSignificantBits()) - .putLong(uuid.getLeastSignificantBits()) - .array(); - } - - /** - * @param uuidSlice textual representation of UUID - * @return byte representation of UUID - * @throws IllegalArgumentException if uuidSlice is not a valid string representation of UUID - */ - public static Slice uuidStringToBytes(Slice uuidSlice) - { - UUID uuid = UUID.fromString(uuidSlice.toStringUtf8()); - byte[] bytes = uuidToBytes(uuid); - return Slices.wrappedBuffer(bytes); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/BaseRaptorConnectorTest.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/BaseRaptorConnectorTest.java deleted file mode 100644 index 9793c30f44dd..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/BaseRaptorConnectorTest.java +++ /dev/null @@ -1,1111 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.SetMultimap; -import io.trino.Session; -import io.trino.spi.type.ArrayType; -import io.trino.testing.BaseConnectorTest; -import io.trino.testing.MaterializedResult; -import io.trino.testing.MaterializedRow; -import io.trino.testing.TestingConnectorBehavior; -import io.trino.testing.sql.TestTable; -import org.intellij.lang.annotations.Language; -import org.junit.jupiter.api.Test; - -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.StringJoiner; -import java.util.UUID; -import java.util.stream.IntStream; - -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.airlift.testing.Assertions.assertLessThan; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.testing.TestingNames.randomNameSuffix; -import static java.lang.String.format; -import static java.util.Arrays.asList; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assumptions.abort; - -public abstract class BaseRaptorConnectorTest - extends BaseConnectorTest -{ - @Override - protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) - { - return switch (connectorBehavior) { - case SUPPORTS_ADD_COLUMN_WITH_COMMENT, - SUPPORTS_COMMENT_ON_COLUMN, - SUPPORTS_COMMENT_ON_TABLE, - SUPPORTS_CREATE_MATERIALIZED_VIEW, - SUPPORTS_CREATE_SCHEMA, - SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT, - SUPPORTS_CREATE_TABLE_WITH_TABLE_COMMENT, - SUPPORTS_NOT_NULL_CONSTRAINT, - SUPPORTS_ROW_TYPE, - SUPPORTS_SET_COLUMN_TYPE, - SUPPORTS_TOPN_PUSHDOWN, - SUPPORTS_TRUNCATE -> false; - default -> super.hasBehavior(connectorBehavior); - }; - } - - @Override - protected TestTable createTableWithDefaultColumns() - { - return abort("Raptor connector does not support column default values"); - } - - @Override - protected void verifyConcurrentUpdateFailurePermissible(Exception e) - { - assertThat(e).hasMessageContaining("Table was updated by a different transaction. Please retry the operation."); - } - - @Test - @Override - public void testCharVarcharComparison() - { - assertThatThrownBy(super::testCharVarcharComparison) - .hasMessage("Unsupported type: char(3)"); - } - - @Test - @Override - public void testRenameTableAcrossSchema() - { - // Raptor allows renaming to a schema it doesn't exist https://github.com/trinodb/trino/issues/11110 - String tableName = "test_rename_old_" + randomNameSuffix(); - assertUpdate("CREATE TABLE " + tableName + " AS SELECT 123 x", 1); - - String schemaName = "test_schema_" + randomNameSuffix(); - - String renamedTable = "test_rename_new_" + randomNameSuffix(); - assertUpdate("ALTER TABLE " + tableName + " RENAME TO " + schemaName + "." + renamedTable); - - assertThat(getQueryRunner().tableExists(getSession(), tableName)).isFalse(); - assertQuery("SELECT x FROM " + schemaName + "." + renamedTable, "VALUES 123"); - - assertUpdate("DROP TABLE " + schemaName + "." + renamedTable); - - assertThat(getQueryRunner().tableExists(getSession(), tableName)).isFalse(); - assertThat(getQueryRunner().tableExists(Session.builder(getSession()).setSchema(schemaName).build(), renamedTable)).isFalse(); - } - - @Override - protected Optional filterDataMappingSmokeTestData(DataMappingTestSetup dataMappingTestSetup) - { - String typeName = dataMappingTestSetup.getTrinoTypeName(); - if (typeName.equals("tinyint") - || typeName.equals("real") - || typeName.startsWith("decimal(") - || typeName.equals("time") - || typeName.equals("time(6)") - || typeName.equals("timestamp(6)") - || typeName.equals("timestamp(3) with time zone") - || typeName.equals("timestamp(6) with time zone") - || typeName.startsWith("char(")) { - //TODO this should either work or fail cleanly - return Optional.empty(); - } - - return Optional.of(dataMappingTestSetup); - } - - @Override - protected Optional filterCaseSensitiveDataMappingTestData(DataMappingTestSetup dataMappingTestSetup) - { - String typeName = dataMappingTestSetup.getTrinoTypeName(); - if (typeName.equals("char(1)")) { - //TODO this should either work or fail cleanly - return Optional.empty(); - } - return Optional.of(dataMappingTestSetup); - } - - @Test - public void testCreateArrayTable() - { - assertUpdate("CREATE TABLE array_test AS SELECT ARRAY [1, 2, 3] AS c", 1); - assertQuery("SELECT cardinality(c) FROM array_test", "SELECT 3"); - assertUpdate("DROP TABLE array_test"); - } - - @Test - public void testMapTable() - { - assertUpdate("CREATE TABLE map_test AS SELECT MAP(ARRAY [1, 2, 3], ARRAY ['hi', 'bye', NULL]) AS c", 1); - assertQuery("SELECT c[1] FROM map_test", "SELECT 'hi'"); - assertQuery("SELECT c[3] FROM map_test", "SELECT NULL"); - assertUpdate("DROP TABLE map_test"); - } - - @Test - @Override - public void testCreateViewSchemaNotFound() - { - // TODO (https://github.com/trinodb/trino/issues/11110) Raptor connector can create new views in a schema where it doesn't exist - assertThatThrownBy(super::testCreateViewSchemaNotFound) - .hasMessageContaining("Expected query to fail: CREATE VIEW test_schema_"); - } - - @Test - public void testCreateTableViewAlreadyExists() - { - assertUpdate("CREATE VIEW view_already_exists AS SELECT 1 a"); - assertQueryFails("CREATE TABLE view_already_exists(a integer)", "View already exists: tpch.view_already_exists"); - assertQueryFails("CREATE TABLE View_Already_Exists(a integer)", "View already exists: tpch.view_already_exists"); - assertQueryFails("CREATE TABLE view_already_exists AS SELECT 1 a", "View already exists: tpch.view_already_exists"); - assertQueryFails("CREATE TABLE View_Already_Exists AS SELECT 1 a", "View already exists: tpch.view_already_exists"); - assertUpdate("DROP VIEW view_already_exists"); - } - - @Test - public void testCreateViewTableAlreadyExists() - { - assertUpdate("CREATE TABLE table_already_exists (id integer)"); - assertQueryFails("CREATE VIEW table_already_exists AS SELECT 1 a", ".*Table already exists: 'raptor.tpch.table_already_exists'"); - assertQueryFails("CREATE VIEW Table_Already_Exists AS SELECT 1 a", ".*Table already exists: 'raptor.tpch.table_already_exists'"); - assertQueryFails("CREATE OR REPLACE VIEW table_already_exists AS SELECT 1 a", ".*Table already exists: 'raptor.tpch.table_already_exists'"); - assertQueryFails("CREATE OR REPLACE VIEW Table_Already_Exists AS SELECT 1 a", ".*Table already exists: 'raptor.tpch.table_already_exists'"); - assertUpdate("DROP TABLE table_already_exists"); - } - - @Test - public void testInsertSelectDecimal() - { - assertUpdate("CREATE TABLE test_decimal(short_decimal DECIMAL(5,2), long_decimal DECIMAL(25,20))"); - assertUpdate("INSERT INTO test_decimal VALUES(DECIMAL '123.45', DECIMAL '12345.12345678901234567890')", "VALUES(1)"); - assertUpdate("INSERT INTO test_decimal VALUES(NULL, NULL)", "VALUES(1)"); - assertQuery("SELECT * FROM test_decimal", "VALUES (123.45, 12345.12345678901234567890), (NULL, NULL)"); - assertUpdate("DROP TABLE test_decimal"); - } - - @Test - public void testShardUuidHiddenColumn() - { - assertUpdate("CREATE TABLE test_shard_uuid AS SELECT orderdate, orderkey FROM orders", "SELECT count(*) FROM orders"); - - MaterializedResult actualResults = computeActual("SELECT *, \"$shard_uuid\" FROM test_shard_uuid"); - assertThat(actualResults.getTypes()).isEqualTo(ImmutableList.of(DATE, BIGINT, SHARD_UUID_COLUMN_TYPE)); - UUID arbitraryUuid = null; - for (MaterializedRow row : actualResults.getMaterializedRows()) { - Object uuid = row.getField(2); - assertInstanceOf(uuid, String.class); - arbitraryUuid = UUID.fromString((String) uuid); - } - assertThat(arbitraryUuid).isNotNull(); - - actualResults = computeActual(format("SELECT * FROM test_shard_uuid where \"$shard_uuid\" = '%s'", arbitraryUuid)); - assertThat(actualResults.getMaterializedRows().size()) - .isNotEqualTo(0); - actualResults = computeActual("SELECT * FROM test_shard_uuid where \"$shard_uuid\" = 'foo'"); - assertThat(actualResults.getMaterializedRows().size()).isEqualTo(0); - } - - @Test - public void testBucketNumberHiddenColumn() - { - assertUpdate("" + - "CREATE TABLE test_bucket_number " + - "WITH (bucket_count = 50, bucketed_on = ARRAY ['orderkey']) " + - "AS SELECT * FROM orders", - "SELECT count(*) FROM orders"); - - MaterializedResult actualResults = computeActual("SELECT DISTINCT \"$bucket_number\" FROM test_bucket_number"); - assertThat(actualResults.getTypes()).isEqualTo(ImmutableList.of(INTEGER)); - Set actual = actualResults.getMaterializedRows().stream() - .map(row -> row.getField(0)) - .collect(toSet()); - assertThat(actual).isEqualTo(IntStream.range(0, 50).boxed().collect(toSet())); - } - - @Test - public void testNoBucketNumberHiddenColumn() - { - assertThatThrownBy(() -> { - assertUpdate("CREATE TABLE test_no_bucket_number (test bigint)"); - computeActual("SELECT DISTINCT \"$bucket_number\" FROM test_no_bucket_number"); - }) - .isInstanceOf(RuntimeException.class) - .hasMessageMatching(".*Column '\\$bucket_number' cannot be resolved"); - } - - @Test - public void testShardingByTemporalDateColumn() - { - // Make sure we have at least 2 different orderdate. - assertThat(computeActual("SELECT count(DISTINCT orderdate) >= 2 FROM orders WHERE orderdate < date '1992-02-08'").getOnlyValue()).isEqualTo(true); - - assertUpdate("CREATE TABLE test_shard_temporal_date " + - "WITH (temporal_column = 'orderdate') AS " + - "SELECT orderdate, orderkey " + - "FROM orders " + - "WHERE orderdate < date '1992-02-08'", - "SELECT count(*) " + - "FROM orders " + - "WHERE orderdate < date '1992-02-08'"); - - MaterializedResult results = computeActual("SELECT orderdate, \"$shard_uuid\" FROM test_shard_temporal_date"); - - // Each shard will only contain data of one date. - SetMultimap shardDateMap = HashMultimap.create(); - for (MaterializedRow row : results.getMaterializedRows()) { - shardDateMap.put((String) row.getField(1), (LocalDate) row.getField(0)); - } - - for (Collection dates : shardDateMap.asMap().values()) { - assertThat(dates.size()).isEqualTo(1); - } - - // Make sure we have all the rows - assertQuery("SELECT orderdate, orderkey FROM test_shard_temporal_date", - "SELECT orderdate, orderkey FROM orders WHERE orderdate < date '1992-02-08'"); - } - - @Test - public void testShardingByTemporalDateColumnBucketed() - { - // Make sure we have at least 2 different orderdate. - assertThat(computeActual("SELECT count(DISTINCT orderdate) >= 2 FROM orders WHERE orderdate < date '1992-02-08'").getOnlyValue()).isEqualTo(true); - - assertUpdate("CREATE TABLE test_shard_temporal_date_bucketed " + - "WITH (temporal_column = 'orderdate', bucket_count = 10, bucketed_on = ARRAY ['orderkey']) AS " + - "SELECT orderdate, orderkey " + - "FROM orders " + - "WHERE orderdate < date '1992-02-08'", - "SELECT count(*) " + - "FROM orders " + - "WHERE orderdate < date '1992-02-08'"); - - MaterializedResult results = computeActual("SELECT orderdate, \"$shard_uuid\" FROM test_shard_temporal_date_bucketed"); - - // Each shard will only contain data of one date. - SetMultimap shardDateMap = HashMultimap.create(); - for (MaterializedRow row : results.getMaterializedRows()) { - shardDateMap.put((String) row.getField(1), (LocalDate) row.getField(0)); - } - - for (Collection dates : shardDateMap.asMap().values()) { - assertThat(dates.size()).isEqualTo(1); - } - - // Make sure we have all the rows - assertQuery("SELECT orderdate, orderkey FROM test_shard_temporal_date_bucketed", - "SELECT orderdate, orderkey FROM orders WHERE orderdate < date '1992-02-08'"); - } - - @Test - public void testShardingByTemporalTimestampColumn() - { - assertUpdate("CREATE TABLE test_shard_temporal_timestamp(col1 BIGINT, col2 TIMESTAMP) WITH (temporal_column = 'col2')"); - - int rows = 20; - StringJoiner joiner = new StringJoiner(", ", "INSERT INTO test_shard_temporal_timestamp VALUES ", ""); - for (int i = 0; i < rows; i++) { - joiner.add(format("(%s, TIMESTAMP '2016-08-08 01:00' + interval '%s' hour)", i, i * 4)); - } - - assertUpdate(joiner.toString(), format("VALUES(%s)", rows)); - - MaterializedResult results = computeActual("SELECT cast(cast(col2 as DATE) as VARCHAR), \"$shard_uuid\" FROM test_shard_temporal_timestamp"); - assertThat(results.getRowCount()).isEqualTo(rows); - - // Each shard will only contain data of one date. - SetMultimap shardDateMap = HashMultimap.create(); - for (MaterializedRow row : results.getMaterializedRows()) { - shardDateMap.put((String) row.getField(1), (String) row.getField(0)); - } - - for (Collection dates : shardDateMap.asMap().values()) { - assertThat(dates.size()).isEqualTo(1); - } - - // Ensure one shard can contain different timestamps from the same day - assertLessThan(shardDateMap.size(), rows); - } - - @Test - public void testShardingByTemporalTimestampColumnBucketed() - { - assertUpdate("" + - "CREATE TABLE test_shard_temporal_timestamp_bucketed(col1 BIGINT, col2 TIMESTAMP) " + - "WITH (temporal_column = 'col2', bucket_count = 3, bucketed_on = ARRAY ['col1'])"); - - int rows = 100; - StringJoiner joiner = new StringJoiner(", ", "INSERT INTO test_shard_temporal_timestamp_bucketed VALUES ", ""); - for (int i = 0; i < rows; i++) { - joiner.add(format("(%s, TIMESTAMP '2016-08-08 01:00' + interval '%s' hour)", i, i)); - } - - assertUpdate(joiner.toString(), format("VALUES(%s)", rows)); - - MaterializedResult results = computeActual("" + - "SELECT cast(cast(col2 as DATE) as VARCHAR), \"$shard_uuid\" " + - "FROM test_shard_temporal_timestamp_bucketed"); - - assertThat(results.getRowCount()).isEqualTo(rows); - - // Each shard will only contain data of one date. - SetMultimap shardDateMap = HashMultimap.create(); - for (MaterializedRow row : results.getMaterializedRows()) { - shardDateMap.put((String) row.getField(1), (String) row.getField(0)); - } - - for (Collection dates : shardDateMap.asMap().values()) { - assertThat(dates.size()).isEqualTo(1); - } - - // Ensure one shard can contain different timestamps from the same day - assertLessThan(shardDateMap.size(), rows); - } - - @Test - public void testTableProperties() - { - computeActual("CREATE TABLE test_table_properties_1 (foo BIGINT, bar BIGINT, ds DATE) WITH (ordering=array['foo','bar'], temporal_column='ds')"); - computeActual("CREATE TABLE test_table_properties_2 (foo BIGINT, bar BIGINT, ds DATE) WITH (ORDERING=array['foo','bar'], TEMPORAL_COLUMN='ds')"); - } - - @Test - public void testShardsSystemTable() - { - assertQuery("" + - "SELECT table_schema, table_name, sum(row_count)\n" + - "FROM system.shards\n" + - "WHERE table_schema = 'tpch'\n" + - " AND table_name IN ('orders', 'region')\n" + - "GROUP BY 1, 2", - "" + - "SELECT 'tpch', 'orders', (SELECT count(*) FROM orders)\n" + - "UNION ALL\n" + - "SELECT 'tpch', 'region', (SELECT count(*) FROM region)"); - } - - @Test - public void testShardsSystemTableWithTemporalColumn() - { - // Make sure we have rows in the selected range - assertThat(computeActual("SELECT count(*) >= 1 FROM orders WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08'").getOnlyValue()).isEqualTo(true); - - // Create a table that has DATE type temporal column - assertUpdate("CREATE TABLE test_shards_system_table_date_temporal\n" + - "WITH (temporal_column = 'orderdate') AS\n" + - "SELECT orderdate, orderkey\n" + - "FROM orders\n" + - "WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08'", - "SELECT count(*)\n" + - "FROM orders\n" + - "WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08'"); - - // Create a table that has TIMESTAMP type temporal column - assertUpdate("CREATE TABLE test_shards_system_table_timestamp_temporal\n" + - "WITH (temporal_column = 'ordertimestamp') AS\n" + - "SELECT CAST (orderdate AS TIMESTAMP) AS ordertimestamp, orderkey\n" + - "FROM test_shards_system_table_date_temporal", - "SELECT count(*)\n" + - "FROM orders\n" + - "WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08'"); - - // For table with DATE type temporal column, min/max_timestamp columns must be null while min/max_date columns must not be null - assertThat(computeActual("" + - "SELECT count(*)\n" + - "FROM system.shards\n" + - "WHERE table_schema = 'tpch'\n" + - "AND table_name = 'test_shards_system_table_date_temporal'\n" + - "AND NOT \n" + - "(min_timestamp IS NULL AND max_timestamp IS NULL\n" + - "AND min_date IS NOT NULL AND max_date IS NOT NULL)").getOnlyValue()).isEqualTo(0L); - - // For table with TIMESTAMP type temporal column, min/max_date columns must be null while min/max_timestamp columns must not be null - assertThat(computeActual("" + - "SELECT count(*)\n" + - "FROM system.shards\n" + - "WHERE table_schema = 'tpch'\n" + - "AND table_name = 'test_shards_system_table_timestamp_temporal'\n" + - "AND NOT\n" + - "(min_date IS NULL AND max_date IS NULL\n" + - "AND min_timestamp IS NOT NULL AND max_timestamp IS NOT NULL)").getOnlyValue()).isEqualTo(0L); - - // Test date predicates in table with DATE temporal column - assertQuery("" + - "SELECT table_schema, table_name, sum(row_count)\n" + - "FROM system.shards \n" + - "WHERE table_schema = 'tpch'\n" + - "AND table_name = 'test_shards_system_table_date_temporal'\n" + - "AND min_date >= date '1992-01-01'\n" + - "AND max_date <= date '1992-02-08'\n" + - "GROUP BY 1, 2", - "" + - "SELECT 'tpch', 'test_shards_system_table_date_temporal',\n" + - "(SELECT count(*) FROM orders WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08')"); - - // Test timestamp predicates in table with TIMESTAMP temporal column - assertQuery("" + - "SELECT table_schema, table_name, sum(row_count)\n" + - "FROM system.shards \n" + - "WHERE table_schema = 'tpch'\n" + - "AND table_name = 'test_shards_system_table_timestamp_temporal'\n" + - "AND min_timestamp >= timestamp '1992-01-01'\n" + - "AND max_timestamp <= timestamp '1992-02-08'\n" + - "GROUP BY 1, 2", - "" + - "SELECT 'tpch', 'test_shards_system_table_timestamp_temporal',\n" + - "(SELECT count(*) FROM orders WHERE orderdate BETWEEN date '1992-01-01' AND date '1992-02-08')"); - } - - @Test - public void testColumnRangesSystemTable() - { - assertQuery("SELECT orderkey_min, orderkey_max, custkey_min, custkey_max, orderdate_min, orderdate_max FROM \"orders$column_ranges\"", - "SELECT min(orderkey), max(orderkey), min(custkey), max(custkey), min(orderdate), max(orderdate) FROM orders"); - - assertQuery("SELECT orderkey_min, orderkey_max FROM \"orders$column_ranges\"", - "SELECT min(orderkey), max(orderkey) FROM orders"); - - // No such table test - assertQueryFails("SELECT * FROM \"no_table$column_ranges\"", ".*'raptor\\.tpch\\.\"no_table\\$column_ranges\"' does not exist.*"); - - // No range column for DOUBLE, INTEGER or VARCHAR - assertQueryFails("SELECT totalprice_min FROM \"orders$column_ranges\"", ".*Column 'totalprice_min' cannot be resolved.*"); - assertQueryFails("SELECT shippriority_min FROM \"orders$column_ranges\"", ".*Column 'shippriority_min' cannot be resolved.*"); - assertQueryFails("SELECT orderstatus_min FROM \"orders$column_ranges\"", ".*Column 'orderstatus_min' cannot be resolved.*"); - assertQueryFails("SELECT orderpriority_min FROM \"orders$column_ranges\"", ".*Column 'orderpriority_min' cannot be resolved.*"); - assertQueryFails("SELECT clerk_min FROM \"orders$column_ranges\"", ".*Column 'clerk_min' cannot be resolved.*"); - assertQueryFails("SELECT comment_min FROM \"orders$column_ranges\"", ".*Column 'comment_min' cannot be resolved.*"); - - // Empty table - assertUpdate("CREATE TABLE column_ranges_test (a BIGINT, b BIGINT)"); - assertQuery("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", "SELECT NULL, NULL, NULL, NULL"); - - // Table with NULL values - assertUpdate("INSERT INTO column_ranges_test VALUES (1, NULL)", 1); - assertQuery("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", "SELECT 1, 1, NULL, NULL"); - assertUpdate("INSERT INTO column_ranges_test VALUES (NULL, 99)", 1); - assertQuery("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", "SELECT 1, 1, 99, 99"); - assertUpdate("INSERT INTO column_ranges_test VALUES (50, 50)", 1); - assertQuery("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", "SELECT 1, 50, 50, 99"); - - // Drop table - assertUpdate("DROP TABLE column_ranges_test"); - assertQueryFails("SELECT a_min, a_max, b_min, b_max FROM \"column_ranges_test$column_ranges\"", - ".*'raptor\\.tpch\\.\"column_ranges_test\\$column_ranges\"' does not exist.*"); - } - - @Test - public void testCreateBucketedTable() - { - assertUpdate("DROP TABLE IF EXISTS orders_bucketed"); - assertUpdate("" + - "CREATE TABLE orders_bucketed " + - "WITH (bucket_count = 50, bucketed_on = ARRAY ['orderkey']) " + - "AS SELECT * FROM orders", - "SELECT count(*) FROM orders"); - - assertQuery("SELECT * FROM orders_bucketed", "SELECT * FROM orders"); - assertQuery("SELECT count(*) FROM orders_bucketed", "SELECT count(*) FROM orders"); - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed", "SELECT 50"); - assertQuery("SELECT count(DISTINCT \"$bucket_number\") FROM orders_bucketed", "SELECT 50"); - - assertUpdate("INSERT INTO orders_bucketed SELECT * FROM orders", "SELECT count(*) FROM orders"); - - assertQuery("SELECT * FROM orders_bucketed", "SELECT * FROM orders UNION ALL SELECT * FROM orders"); - assertQuery("SELECT count(*) FROM orders_bucketed", "SELECT count(*) * 2 FROM orders"); - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed", "SELECT 50 * 2"); - assertQuery("SELECT count(DISTINCT \"$bucket_number\") FROM orders_bucketed", "SELECT 50"); - - assertQuery("SELECT count(*) FROM orders_bucketed a JOIN orders_bucketed b USING (orderkey)", "SELECT count(*) * 4 FROM orders"); - - assertUpdate("DELETE FROM orders_bucketed WHERE orderkey = 37", 2); - assertQuery("SELECT count(*) FROM orders_bucketed", "SELECT (count(*) * 2) - 2 FROM orders"); - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed", "SELECT 50 * 2"); - assertQuery("SELECT count(DISTINCT \"$bucket_number\") FROM orders_bucketed", "SELECT 50"); - - assertUpdate("DROP TABLE orders_bucketed"); - } - - @Test - public void testCreateBucketedTableLike() - { - assertUpdate("" + - "CREATE TABLE orders_bucketed_original (" + - " orderkey bigint" + - ", custkey bigint" + - ") " + - "WITH (bucket_count = 50, bucketed_on = ARRAY['orderkey'])"); - - assertUpdate("" + - "CREATE TABLE orders_bucketed_like (" + - " orderdate date" + - ", LIKE orders_bucketed_original INCLUDING PROPERTIES" + - ")"); - - assertUpdate("INSERT INTO orders_bucketed_like SELECT orderdate, orderkey, custkey FROM orders", "SELECT count(*) FROM orders"); - assertUpdate("INSERT INTO orders_bucketed_like SELECT orderdate, orderkey, custkey FROM orders", "SELECT count(*) FROM orders"); - - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed_like", "SELECT 50 * 2"); - - assertUpdate("DROP TABLE orders_bucketed_original"); - assertUpdate("DROP TABLE orders_bucketed_like"); - } - - @Test - public void testBucketingMixedTypes() - { - assertUpdate("" + - "CREATE TABLE orders_bucketed_mixed " + - "WITH (bucket_count = 50, bucketed_on = ARRAY ['custkey', 'clerk', 'shippriority']) " + - "AS SELECT * FROM orders", - "SELECT count(*) FROM orders"); - - assertQuery("SELECT * FROM orders_bucketed_mixed", "SELECT * FROM orders"); - assertQuery("SELECT count(*) FROM orders_bucketed_mixed", "SELECT count(*) FROM orders"); - assertQuery("SELECT count(DISTINCT \"$shard_uuid\") FROM orders_bucketed_mixed", "SELECT 50"); - assertQuery("SELECT count(DISTINCT \"$bucket_number\") FROM orders_bucketed_mixed", "SELECT 50"); - } - - @Test - @Override - public void testShowCreateTable() - { - super.testShowCreateTable(); - - String createTableSql = format("" + - "CREATE TABLE %s.%s.%s (\n" + - " c1 bigint,\n" + - " c2 double,\n" + - " \"c 3\" varchar,\n" + - " \"c'4\" array(bigint),\n" + - " c5 map(bigint, varchar),\n" + - " c6 bigint,\n" + - " c7 timestamp(3)\n" + - ")\n" + - "WITH (\n" + - " bucket_count = 32,\n" + - " bucketed_on = ARRAY['c1','c6'],\n" + - " ordering = ARRAY['c6','c1'],\n" + - " temporal_column = 'c7'\n" + - ")", - getSession().getCatalog().get(), getSession().getSchema().get(), "test_show_create_table"); - assertUpdate(createTableSql); - - MaterializedResult actualResult = computeActual("SHOW CREATE TABLE test_show_create_table"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE " + getSession().getSchema().get() + ".test_show_create_table"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE " + getSession().getCatalog().get() + "." + getSession().getSchema().get() + ".test_show_create_table"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - // With organization enabled - createTableSql = format("" + - "CREATE TABLE %s.%s.%s (\n" + - " c1 bigint,\n" + - " c2 double,\n" + - " \"c 3\" varchar,\n" + - " \"c'4\" array(bigint),\n" + - " c5 map(bigint, varchar),\n" + - " c6 bigint,\n" + - " c7 timestamp(3)\n" + - ")\n" + - "WITH (\n" + - " bucket_count = 32,\n" + - " bucketed_on = ARRAY['c1','c6'],\n" + - " ordering = ARRAY['c6','c1'],\n" + - " organized = true\n" + - ")", - getSession().getCatalog().get(), getSession().getSchema().get(), "test_show_create_table_organized"); - assertUpdate(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE test_show_create_table_organized"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE " + getSession().getSchema().get() + ".test_show_create_table_organized"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE " + getSession().getCatalog().get() + "." + getSession().getSchema().get() + ".test_show_create_table_organized"); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - - createTableSql = format("" + - "CREATE TABLE %s.%s.%s (\n" + - " \"c\"\"1\" bigint,\n" + - " c2 double,\n" + - " \"c 3\" varchar,\n" + - " \"c'4\" array(bigint),\n" + - " c5 map(bigint, varchar)\n" + - ")", - getSession().getCatalog().get(), getSession().getSchema().get(), "\"test_show_create_table\"\"2\""); - assertUpdate(createTableSql); - - actualResult = computeActual("SHOW CREATE TABLE \"test_show_create_table\"\"2\""); - assertThat(getOnlyElement(actualResult.getOnlyColumnAsSet())).isEqualTo(createTableSql); - } - - @Test - @Override - public void testCreateTableSchemaNotFound() - { - // TODO (https://github.com/trinodb/trino/issues/11110) Raptor connector can create new tables in a schema where it doesn't exist - assertThatThrownBy(super::testCreateTableSchemaNotFound) - .hasMessageContaining("Expected query to fail: CREATE TABLE test_schema_"); - } - - @Test - @Override - public void testCreateTableAsSelectSchemaNotFound() - { - // TODO (https://github.com/trinodb/trino/issues/11110) Raptor connector can create new tables in a schema where it doesn't exist - assertThatThrownBy(super::testCreateTableAsSelectSchemaNotFound) - .hasMessageContaining("Expected query to fail: CREATE TABLE test_schema_"); - } - - @Test - public void testTablesSystemTable() - { - assertUpdate("" + - "CREATE TABLE system_tables_test0 (c00 timestamp, c01 varchar, c02 double, c03 bigint, c04 bigint)"); - assertUpdate("" + - "CREATE TABLE system_tables_test1 (c10 timestamp, c11 varchar, c12 double, c13 bigint, c14 bigint) " + - "WITH (temporal_column = 'c10')"); - assertUpdate("" + - "CREATE TABLE system_tables_test2 (c20 timestamp, c21 varchar, c22 double, c23 bigint, c24 bigint) " + - "WITH (temporal_column = 'c20', ordering = ARRAY['c22', 'c21'])"); - assertUpdate("" + - "CREATE TABLE system_tables_test3 (c30 timestamp, c31 varchar, c32 double, c33 bigint, c34 bigint) " + - "WITH (temporal_column = 'c30', bucket_count = 40, bucketed_on = ARRAY ['c34', 'c33'])"); - assertUpdate("" + - "CREATE TABLE system_tables_test4 (c40 timestamp, c41 varchar, c42 double, c43 bigint, c44 bigint) " + - "WITH (temporal_column = 'c40', ordering = ARRAY['c41', 'c42'], distribution_name = 'test_distribution', bucket_count = 50, bucketed_on = ARRAY ['c43', 'c44'])"); - assertUpdate("" + - "CREATE TABLE system_tables_test5 (c50 timestamp, c51 varchar, c52 double, c53 bigint, c54 bigint) " + - "WITH (ordering = ARRAY['c51', 'c52'], distribution_name = 'test_distribution', bucket_count = 50, bucketed_on = ARRAY ['c53', 'c54'], organized = true)"); - - MaterializedResult actualResults = computeActual("SELECT * FROM system.tables"); - assertThat(actualResults.getTypes()).isEqualTo(ImmutableList.builder() - .add(VARCHAR) // table_schema - .add(VARCHAR) // table_name - .add(VARCHAR) // temporal_column - .add(new ArrayType(VARCHAR)) // ordering_columns - .add(VARCHAR) // distribution_name - .add(BIGINT) // bucket_count - .add(new ArrayType(VARCHAR)) // bucket_columns - .add(BOOLEAN) // organized - .build()); - Map map = actualResults.getMaterializedRows().stream() - .filter(row -> ((String) row.getField(1)).startsWith("system_tables_test")) - .collect(toImmutableMap(row -> ((String) row.getField(1)), identity())); - assertThat(map.size()).isEqualTo(6); - assertThat(map.get("system_tables_test0").getFields()).isEqualTo(asList("tpch", "system_tables_test0", null, null, null, null, null, Boolean.FALSE)); - assertThat(map.get("system_tables_test1").getFields()).isEqualTo(asList("tpch", "system_tables_test1", "c10", null, null, null, null, Boolean.FALSE)); - assertThat(map.get("system_tables_test2").getFields()).isEqualTo(asList("tpch", "system_tables_test2", "c20", ImmutableList.of("c22", "c21"), null, null, null, Boolean.FALSE)); - assertThat(map.get("system_tables_test3").getFields()).isEqualTo(asList("tpch", "system_tables_test3", "c30", null, null, 40L, ImmutableList.of("c34", "c33"), Boolean.FALSE)); - assertThat(map.get("system_tables_test4").getFields()).isEqualTo(asList("tpch", "system_tables_test4", "c40", ImmutableList.of("c41", "c42"), "test_distribution", 50L, ImmutableList.of("c43", "c44"), Boolean.FALSE)); - assertThat(map.get("system_tables_test5").getFields()).isEqualTo(asList("tpch", "system_tables_test5", null, ImmutableList.of("c51", "c52"), "test_distribution", 50L, ImmutableList.of("c53", "c54"), Boolean.TRUE)); - - actualResults = computeActual("SELECT * FROM system.tables WHERE table_schema = 'tpch'"); - long actualRowCount = actualResults.getMaterializedRows().stream() - .filter(row -> ((String) row.getField(1)).startsWith("system_tables_test")) - .count(); - assertThat(actualRowCount).isEqualTo(6); - - actualResults = computeActual("SELECT * FROM system.tables WHERE table_name = 'system_tables_test3'"); - assertThat(actualResults.getMaterializedRows().size()).isEqualTo(1); - - actualResults = computeActual("SELECT * FROM system.tables WHERE table_schema = 'tpch' and table_name = 'system_tables_test3'"); - assertThat(actualResults.getMaterializedRows().size()).isEqualTo(1); - - actualResults = computeActual("" + - "SELECT distribution_name, bucket_count, bucketing_columns, ordering_columns, temporal_column, organized " + - "FROM system.tables " + - "WHERE table_schema = 'tpch' and table_name = 'system_tables_test3'"); - assertThat(actualResults.getTypes()).isEqualTo(ImmutableList.of(VARCHAR, BIGINT, new ArrayType(VARCHAR), new ArrayType(VARCHAR), VARCHAR, BOOLEAN)); - assertThat(actualResults.getMaterializedRows().size()).isEqualTo(1); - - assertUpdate("DROP TABLE system_tables_test0"); - assertUpdate("DROP TABLE system_tables_test1"); - assertUpdate("DROP TABLE system_tables_test2"); - assertUpdate("DROP TABLE system_tables_test3"); - assertUpdate("DROP TABLE system_tables_test4"); - assertUpdate("DROP TABLE system_tables_test5"); - - assertThat(computeActual("SELECT * FROM system.tables WHERE table_schema IN ('foo', 'bar')").getRowCount()).isEqualTo(0); - } - - @Test - public void testTableStatsSystemTable() - { - // basic sanity tests - assertQuery("" + - "SELECT table_schema, table_name, sum(row_count)\n" + - "FROM system.table_stats\n" + - "WHERE table_schema = 'tpch'\n" + - " AND table_name IN ('orders', 'region')\n" + - "GROUP BY 1, 2", - "" + - "SELECT 'tpch', 'orders', (SELECT count(*) FROM orders)\n" + - "UNION ALL\n" + - "SELECT 'tpch', 'region', (SELECT count(*) FROM region)"); - - assertQuery("" + - "SELECT\n" + - " bool_and(row_count >= shard_count)\n" + - ", bool_and(update_time >= create_time)\n" + - ", bool_and(table_version >= 1)\n" + - "FROM system.table_stats\n" + - "WHERE row_count > 0", - "SELECT true, true, true"); - - // create empty table - assertUpdate("CREATE TABLE test_table_stats (x bigint)"); - - @Language("SQL") String sql = "" + - "SELECT create_time, update_time, table_version," + - " shard_count, row_count, uncompressed_size\n" + - "FROM system.table_stats\n" + - "WHERE table_schema = 'tpch'\n" + - " AND table_name = 'test_table_stats'"; - MaterializedRow row = getOnlyElement(computeActual(sql).getMaterializedRows()); - - LocalDateTime createTime = (LocalDateTime) row.getField(0); - LocalDateTime updateTime1 = (LocalDateTime) row.getField(1); - assertThat(createTime).isEqualTo(updateTime1); - - assertThat(row.getField(2)).isEqualTo(1L); // table_version - assertThat(row.getField(3)).isEqualTo(0L); // shard_count - assertThat(row.getField(4)).isEqualTo(0L); // row_count - long size1 = (long) row.getField(5); // uncompressed_size - - // insert - assertUpdate("INSERT INTO test_table_stats VALUES (1), (2), (3), (4)", 4); - row = getOnlyElement(computeActual(sql).getMaterializedRows()); - - assertThat(row.getField(0)).isEqualTo(createTime); - LocalDateTime updateTime2 = (LocalDateTime) row.getField(1); - assertLessThan(updateTime1, updateTime2); - - assertThat(row.getField(2)).isEqualTo(2L); // table_version - assertGreaterThanOrEqual((Long) row.getField(3), 1L); // shard_count - assertThat(row.getField(4)).isEqualTo(4L); // row_count - long size2 = (long) row.getField(5); // uncompressed_size - assertGreaterThan(size2, size1); - - // delete - assertUpdate("DELETE FROM test_table_stats WHERE x IN (2, 4)", 2); - row = getOnlyElement(computeActual(sql).getMaterializedRows()); - - assertThat(row.getField(0)).isEqualTo(createTime); - LocalDateTime updateTime3 = (LocalDateTime) row.getField(1); - assertLessThan(updateTime2, updateTime3); - - assertThat(row.getField(2)).isEqualTo(3L); // table_version - assertGreaterThanOrEqual((Long) row.getField(3), 1L); // shard_count - assertThat(row.getField(4)).isEqualTo(2L); // row_count - long size3 = (long) row.getField(5); // uncompressed_Size - assertLessThan(size3, size2); - - // add column - assertUpdate("ALTER TABLE test_table_stats ADD COLUMN y bigint"); - row = getOnlyElement(computeActual(sql).getMaterializedRows()); - - assertThat(row.getField(0)).isEqualTo(createTime); - assertLessThan(updateTime3, (LocalDateTime) row.getField(1)); - - assertThat(row.getField(2)).isEqualTo(4L); // table_version - assertThat(row.getField(4)).isEqualTo(2L); // row_count - assertThat(row.getField(5)).isEqualTo(size3); // uncompressed_size - - // cleanup - assertUpdate("DROP TABLE test_table_stats"); - } - - @Test - public void testAlterTable() - { - assertUpdate("CREATE TABLE test_alter_table (c1 bigint, c2 bigint)"); - assertUpdate("INSERT INTO test_alter_table VALUES (1, 1), (1, 2), (1, 3), (1, 4)", 4); - assertUpdate("INSERT INTO test_alter_table VALUES (11, 1), (11, 2)", 2); - - assertUpdate("ALTER TABLE test_alter_table ADD COLUMN c3 bigint"); - assertQueryFails("ALTER TABLE test_alter_table DROP COLUMN c3", "Cannot drop the column which has the largest column ID in the table"); - assertUpdate("INSERT INTO test_alter_table VALUES (2, 1, 1), (2, 2, 2), (2, 3, 3), (2, 4, 4)", 4); - assertUpdate("INSERT INTO test_alter_table VALUES (22, 1, 1), (22, 2, 2), (22, 4, 4)", 3); - - // Do a partial delete on a shard that does not contain newly added column - assertUpdate("DELETE FROM test_alter_table WHERE c1 = 1 and c2 = 1", 1); - // Then drop a full shard that does not contain newly added column - assertUpdate("DELETE FROM test_alter_table WHERE c1 = 11", 2); - - // Drop a column from middle of table - assertUpdate("ALTER TABLE test_alter_table DROP COLUMN c2"); - assertUpdate("INSERT INTO test_alter_table VALUES (3, 1), (3, 2), (3, 3), (3, 4)", 4); - - // Do a partial delete on a shard that contains column already dropped - assertUpdate("DELETE FROM test_alter_table WHERE c1 = 2 and c3 = 1", 1); - // Then drop a full shard that contains column already dropped - assertUpdate("DELETE FROM test_alter_table WHERE c1 = 22", 3); - - assertUpdate("DROP TABLE test_alter_table"); - } - - @Override - protected void verifyConcurrentAddColumnFailurePermissible(Exception e) - { - assertThat(e) - .hasMessageContaining("Failed to perform metadata operation") - .cause() - .hasMessageMatching( - "(?s).*SQLIntegrityConstraintViolationException.*" + - "|.*Unique index or primary key violation.*" + - "|.*Deadlock found when trying to get lock; try restarting transaction.*"); - } - - @Override - protected OptionalInt maxTableNameLength() - { - return OptionalInt.of(255); - } - - @Override - protected void verifyTableNameLengthFailurePermissible(Throwable e) - { - assertThat(e).hasMessage("Failed to perform metadata operation"); - } - - @Override - protected OptionalInt maxColumnNameLength() - { - return OptionalInt.of(255); - } - - @Override - protected void verifyColumnNameLengthFailurePermissible(Throwable e) - { - assertThat(e).hasMessage("Failed to perform metadata operation"); - } - - @Test - public void testMergeMultipleOperationsUnbucketed() - { - String targetTable = "merge_multiple_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (customer VARCHAR, purchases INT, zipcode INT, spouse VARCHAR, address VARCHAR)", targetTable)); - testMergeMultipleOperationsInternal(targetTable, 32); - } - - @Test - public void testMergeMultipleOperationsBucketed() - { - String targetTable = "merge_multiple_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (customer VARCHAR, purchases INT, zipcode INT, spouse VARCHAR, address VARCHAR)" + - " WITH (bucket_count=4, bucketed_on=ARRAY['customer'])", targetTable)); - testMergeMultipleOperationsInternal(targetTable, 32); - } - - private void testMergeMultipleOperationsInternal(String targetTable, int targetCustomerCount) - { - String originalInsertFirstHalf = IntStream.range(1, targetCustomerCount / 2) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jan_%s', '%s Poe Ct')", intValue, 1000, 91000, intValue, intValue)) - .collect(joining(", ")); - String originalInsertSecondHalf = IntStream.range(targetCustomerCount / 2, targetCustomerCount) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jan_%s', '%s Poe Ct')", intValue, 2000, 92000, intValue, intValue)) - .collect(joining(", ")); - - assertUpdate(format("INSERT INTO %s (customer, purchases, zipcode, spouse, address) VALUES %s, %s", targetTable, originalInsertFirstHalf, originalInsertSecondHalf), targetCustomerCount - 1); - - String firstMergeSource = IntStream.range(targetCustomerCount / 2, targetCustomerCount) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jill_%s', '%s Eop Ct')", intValue, 3000, 83000, intValue, intValue)) - .collect(joining(", ")); - - @Language("SQL") String sql = format("MERGE INTO %s t USING (SELECT * FROM (VALUES %s)) AS s(customer, purchases, zipcode, spouse, address)", targetTable, firstMergeSource) + - " ON t.customer = s.customer" + - " WHEN MATCHED THEN UPDATE SET purchases = s.purchases, zipcode = s.zipcode, spouse = s.spouse, address = s.address"; - assertUpdate(sql, targetCustomerCount / 2); - - assertQuery( - "SELECT customer, purchases, zipcode, spouse, address FROM " + targetTable, - format("SELECT * FROM (VALUES %s, %s) AS v(customer, purchases, zipcode, spouse, address)", originalInsertFirstHalf, firstMergeSource)); - - String nextInsert = IntStream.range(targetCustomerCount, targetCustomerCount * 3 / 2) - .mapToObj(intValue -> format("('jack_%s', %s, %s, 'jan_%s', '%s Poe Ct')", intValue, 4000, 74000, intValue, intValue)) - .collect(joining(", ")); - assertUpdate(format("INSERT INTO %s (customer, purchases, zipcode, spouse, address) VALUES %s", targetTable, nextInsert), targetCustomerCount / 2); - - String secondMergeSource = IntStream.range(1, targetCustomerCount * 3 / 2) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jen_%s', '%s Poe Ct')", intValue, 5000, 85000, intValue, intValue)) - .collect(joining(", ")); - - assertUpdate(format("MERGE INTO %s t USING (SELECT * FROM (VALUES %s)) AS s(customer, purchases, zipcode, spouse, address)", targetTable, secondMergeSource) + - " ON t.customer = s.customer" + - " WHEN MATCHED AND t.zipcode = 91000 THEN DELETE" + - " WHEN MATCHED AND s.zipcode = 85000 THEN UPDATE SET zipcode = 60000" + - " WHEN MATCHED THEN UPDATE SET zipcode = s.zipcode, spouse = s.spouse, address = s.address" + - " WHEN NOT MATCHED THEN INSERT (customer, purchases, zipcode, spouse, address) VALUES(s.customer, s.purchases, s.zipcode, s.spouse, s.address)", - targetCustomerCount * 3 / 2 - 1); - - String updatedBeginning = IntStream.range(targetCustomerCount / 2, targetCustomerCount) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jill_%s', '%s Eop Ct')", intValue, 3000, 60000, intValue, intValue)) - .collect(joining(", ")); - String updatedMiddle = IntStream.range(targetCustomerCount, targetCustomerCount * 3 / 2) - .mapToObj(intValue -> format("('joe_%s', %s, %s, 'jen_%s', '%s Poe Ct')", intValue, 5000, 85000, intValue, intValue)) - .collect(joining(", ")); - String updatedEnd = IntStream.range(targetCustomerCount, targetCustomerCount * 3 / 2) - .mapToObj(intValue -> format("('jack_%s', %s, %s, 'jan_%s', '%s Poe Ct')", intValue, 4000, 74000, intValue, intValue)) - .collect(joining(", ")); - - assertQuery( - "SELECT customer, purchases, zipcode, spouse, address FROM " + targetTable, - format("SELECT * FROM (VALUES %s, %s, %s) AS v(customer, purchases, zipcode, spouse, address)", updatedBeginning, updatedMiddle, updatedEnd)); - assertUpdate("DROP TABLE " + targetTable); - } - - @Test - public void testMergeSimpleQueryBucketed() - { - String targetTable = "merge_simple_target_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count=7, bucketed_on=ARRAY['address'])", targetTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 5, 'Antioch'), ('Bill', 7, 'Buena'), ('Carol', 3, 'Cambridge'), ('Dave', 11, 'Devon')", targetTable), 4); - - @Language("SQL") String query = format("MERGE INTO %s t USING ", targetTable) + - "(SELECT * FROM (VALUES ('Aaron', 6, 'Arches'), ('Carol', 9, 'Centreville'), ('Dave', 11, 'Darbyshire'), ('Ed', 7, 'Etherville'))) AS s(customer, purchases, address)" + - " " + - "ON (t.customer = s.customer)" + - " WHEN MATCHED AND s.address = 'Centreville' THEN DELETE" + - " WHEN MATCHED THEN UPDATE SET purchases = s.purchases + t.purchases, address = s.address" + - " WHEN NOT MATCHED THEN INSERT (customer, purchases, address) VALUES(s.customer, s.purchases, s.address)"; - assertUpdate(query, 4); - - assertQuery("SELECT * FROM " + targetTable, "VALUES ('Aaron', 11, 'Arches'), ('Bill', 7, 'Buena'), ('Dave', 22, 'Darbyshire'), ('Ed', 7, 'Etherville')"); - } - - @Test - public void testMergeMultipleRows() - { - testMergeMultipleRowsMatchFails("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR)"); - testMergeMultipleRowsMatchFails("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['customer'])"); - testMergeMultipleRowsMatchFails("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 4, bucketed_on = ARRAY['address'])"); - testMergeMultipleRowsMatchFails("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 4, bucketed_on = ARRAY['address', 'purchases', 'customer'])"); - } - - private void testMergeMultipleRowsMatchFails(String createTableSql) - { - String targetTable = "merge_all_matches_deleted_target_" + randomNameSuffix(); - assertUpdate(format(createTableSql, targetTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 5, 'Antioch'), ('Bill', 7, 'Antioch')", targetTable), 2); - - String sourceTable = "merge_all_matches_deleted_source_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR)", sourceTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 6, 'Adelphi'), ('Aaron', 8, 'Ashland')", sourceTable), 2); - - assertThatThrownBy(() -> computeActual(format("MERGE INTO %s t USING %s s ON (t.customer = s.customer)", targetTable, sourceTable) + - " WHEN MATCHED THEN UPDATE SET address = s.address")) - .hasMessage("One MERGE target table row matched more than one source row"); - - assertUpdate(format("MERGE INTO %s t USING %s s ON (t.customer = s.customer)", targetTable, sourceTable) + - " WHEN MATCHED AND s.address = 'Adelphi' THEN UPDATE SET address = s.address", - 1); - assertQuery("SELECT customer, purchases, address FROM " + targetTable, "VALUES ('Aaron', 5, 'Adelphi'), ('Bill', 7, 'Antioch')"); - - assertUpdate("DROP TABLE " + sourceTable); - assertUpdate("DROP TABLE " + targetTable); - } - - @Test - public void testMergeWithDifferentBucketing() - { - testMergeWithDifferentBucketing( - "target_and_source_with_different_bucketing_counts", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 5, bucketed_on = ARRAY['customer'])", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['purchases', 'address'])"); - testMergeWithDifferentBucketing( - "target_and_source_with_different_bucketing_columns", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['address'])", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['customer'])"); - testMergeWithDifferentBucketing( - "target_flat_source_bucketed_by_customer", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR)", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['customer'])"); - testMergeWithDifferentBucketing( - "target_bucketed_by_customer_source_flat", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR) WITH (bucket_count = 3, bucketed_on = ARRAY['customer'])", - "CREATE TABLE %s (customer VARCHAR, purchases INT, address VARCHAR)"); - } - - private void testMergeWithDifferentBucketing(String testDescription, String createTargetTableSql, String createSourceTableSql) - { - String targetTable = format("%s_target_%s", testDescription, randomNameSuffix()); - assertUpdate(format(createTargetTableSql, targetTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 5, 'Antioch'), ('Bill', 7, 'Buena'), ('Carol', 3, 'Cambridge'), ('Dave', 11, 'Devon')", targetTable), 4); - - String sourceTable = format("%s_source_%s", testDescription, randomNameSuffix()); - assertUpdate(format(createSourceTableSql, sourceTable)); - - assertUpdate(format("INSERT INTO %s (customer, purchases, address) VALUES ('Aaron', 6, 'Arches'), ('Ed', 7, 'Etherville'), ('Carol', 9, 'Centreville'), ('Dave', 11, 'Darbyshire')", sourceTable), 4); - - @Language("SQL") String sql = format("MERGE INTO %s t USING %s s ON (t.customer = s.customer)", targetTable, sourceTable) + - " WHEN MATCHED AND s.address = 'Centreville' THEN DELETE" + - " WHEN MATCHED THEN UPDATE SET purchases = s.purchases + t.purchases, address = s.address" + - " WHEN NOT MATCHED THEN INSERT (customer, purchases, address) VALUES(s.customer, s.purchases, s.address)"; - - assertUpdate(sql, 4); - - assertQuery("SELECT customer, purchases, address FROM " + targetTable, "VALUES ('Aaron', 11, 'Arches'), ('Ed', 7, 'Etherville'), ('Bill', 7, 'Buena'), ('Dave', 22, 'Darbyshire')"); - - assertUpdate("DROP TABLE " + sourceTable); - assertUpdate("DROP TABLE " + targetTable); - } - - @Test - public void testMergeOverManySplits() - { - String targetTable = "merge_delete_select_" + randomNameSuffix(); - assertUpdate(format("CREATE TABLE %s (orderkey bigint, custkey bigint, orderstatus varchar(1), totalprice double, orderdate date, orderpriority varchar(15), clerk varchar(15), shippriority integer, comment varchar(79))", targetTable)); - - assertUpdate(format("INSERT INTO %s SELECT * FROM tpch.\"sf0.1\".orders", targetTable), 150000); - - @Language("SQL") String sql = format("MERGE INTO %s t USING (SELECT * FROM tpch.\"sf0.1\".orders) s ON (t.orderkey = s.orderkey)", targetTable) + - " WHEN MATCHED AND mod(s.orderkey, 3) = 0 THEN UPDATE SET totalprice = t.totalprice + s.totalprice" + - " WHEN MATCHED AND mod(s.orderkey, 3) = 1 THEN DELETE"; - - assertUpdate(sql, 100_000); - - assertQuery(format("SELECT count(*) FROM %s t WHERE mod(t.orderkey, 3) = 1", targetTable), "VALUES (0)"); - - assertUpdate("DROP TABLE " + targetTable); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/DatabaseTesting.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/DatabaseTesting.java deleted file mode 100644 index 1ce2b73256de..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/DatabaseTesting.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.sqlobject.SqlObjectPlugin; - -import java.util.concurrent.ThreadLocalRandom; - -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; - -public final class DatabaseTesting -{ - private DatabaseTesting() {} - - public static Jdbi createTestingJdbi() - { - return Jdbi.create("jdbc:h2:mem:test" + System.nanoTime() + ThreadLocalRandom.current().nextLong()) - .installPlugin(new SqlObjectPlugin()) - .registerRowMapper(new TableColumn.Mapper(TESTING_TYPE_MANAGER)) - .registerRowMapper(new Distribution.Mapper(TESTING_TYPE_MANAGER)); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/RaptorQueryRunner.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/RaptorQueryRunner.java deleted file mode 100644 index d0e567f220d3..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/RaptorQueryRunner.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.errorprone.annotations.CanIgnoreReturnValue; -import io.airlift.log.Logger; -import io.trino.Session; -import io.trino.SystemSessionProperties; -import io.trino.connector.CatalogServiceProvider; -import io.trino.metadata.QualifiedObjectName; -import io.trino.metadata.SessionPropertyManager; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.plugin.tpch.TpchPlugin; -import io.trino.spi.session.PropertyMetadata; -import io.trino.testing.DistributedQueryRunner; -import io.trino.testing.QueryRunner; -import io.trino.tpch.TpchTable; -import org.intellij.lang.annotations.Language; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import static io.airlift.testing.Closeables.closeAllSuppress; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.trino.testing.QueryAssertions.copyTpchTables; -import static io.trino.testing.TestingHandles.createTestCatalogHandle; -import static io.trino.testing.TestingSession.testSessionBuilder; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public final class RaptorQueryRunner -{ - private RaptorQueryRunner() {} - - private static final Logger log = Logger.get(RaptorQueryRunner.class); - - public static Builder builder() - { - return new Builder() - .addConnectorProperty("metadata.db.type", "h2") - .addConnectorProperty("metadata.db.filename", createTempDirectory("raptor-db").toString()) - .addConnectorProperty("storage.data-directory", createTempDirectory("raptor-data").toString()) - .addConnectorProperty("storage.max-shard-rows", "2000") - .addConnectorProperty("backup.provider", "file") - .addConnectorProperty("backup.directory", createTempDirectory("raptor-backup").toString()); - } - - public static final class Builder - extends DistributedQueryRunner.Builder - { - private final Map connectorProperties = new HashMap<>(); - private boolean bucketed; - private List> initialTables = ImmutableList.of(); - - private Builder() - { - super(createSession()); - } - - @CanIgnoreReturnValue - public Builder addConnectorProperty(String key, String value) - { - this.connectorProperties.put(key, value); - return this; - } - - @CanIgnoreReturnValue - public Builder enableBucketed() - { - this.bucketed = true; - return this; - } - - @CanIgnoreReturnValue - public Builder setInitialTables(Iterable> initialTables) - { - this.initialTables = ImmutableList.copyOf(requireNonNull(initialTables, "initialTables is null")); - return this; - } - - @Override - public DistributedQueryRunner build() - throws Exception - { - DistributedQueryRunner queryRunner = super.build(); - try { - queryRunner.installPlugin(new TpchPlugin()); - queryRunner.createCatalog("tpch", "tpch"); - - queryRunner.installPlugin(new RaptorPlugin()); - queryRunner.createCatalog("raptor", "raptor_legacy", connectorProperties); - - copyTables(queryRunner, "tpch", createSession(), bucketed, initialTables); - - return queryRunner; - } - catch (Throwable e) { - closeAllSuppress(e, queryRunner); - throw e; - } - } - } - - public static void copyTables(QueryRunner queryRunner, String catalog, Session session, boolean bucketed, List> tables) - { - String schema = TINY_SCHEMA_NAME; - if (!bucketed) { - copyTpchTables(queryRunner, catalog, schema, session, tables); - return; - } - - ImmutableMap.Builder, String> tablesMapBuilder = ImmutableMap.builder(); - for (TpchTable table : tables) { - if (table.equals(TpchTable.ORDERS)) { - tablesMapBuilder.put(TpchTable.ORDERS, "bucket_count = 25, bucketed_on = ARRAY['orderkey'], distribution_name = 'order'"); - } - else if (table.equals(TpchTable.LINE_ITEM)) { - tablesMapBuilder.put(TpchTable.LINE_ITEM, "bucket_count = 25, bucketed_on = ARRAY['orderkey'], distribution_name = 'order'"); - } - else if (table.equals(TpchTable.PART)) { - tablesMapBuilder.put(TpchTable.PART, "bucket_count = 20, bucketed_on = ARRAY['partkey'], distribution_name = 'part'"); - } - else if (table.equals(TpchTable.PART_SUPPLIER)) { - tablesMapBuilder.put(TpchTable.PART_SUPPLIER, "bucket_count = 20, bucketed_on = ARRAY['partkey'], distribution_name = 'part'"); - } - else if (table.equals(TpchTable.SUPPLIER)) { - tablesMapBuilder.put(TpchTable.SUPPLIER, "bucket_count = 10, bucketed_on = ARRAY['suppkey']"); - } - else if (table.equals(TpchTable.CUSTOMER)) { - tablesMapBuilder.put(TpchTable.CUSTOMER, "bucket_count = 10, bucketed_on = ARRAY['custkey']"); - } - else if (table.equals(TpchTable.NATION)) { - tablesMapBuilder.put(TpchTable.NATION, ""); - } - else if (table.equals(TpchTable.REGION)) { - tablesMapBuilder.put(TpchTable.REGION, ""); - } - else { - throw new IllegalArgumentException("Unsupported table: " + table); - } - } - Map, String> tablesMap = tablesMapBuilder.buildOrThrow(); - - log.info("Loading data from %s.%s...", catalog, schema); - long startTime = System.nanoTime(); - for (Entry, String> entry : tablesMap.entrySet()) { - copyTable(queryRunner, catalog, session, schema, entry.getKey(), entry.getValue()); - } - log.info("Loading from %s.%s complete in %s", catalog, schema, nanosSince(startTime)); - } - - private static void copyTable(QueryRunner queryRunner, String catalog, Session session, String schema, TpchTable table, String properties) - { - QualifiedObjectName source = new QualifiedObjectName(catalog, schema, table.getTableName()); - String target = table.getTableName(); - - String with = properties.isEmpty() ? "" : format(" WITH (%s)", properties); - @Language("SQL") String sql = format("CREATE TABLE %s%s AS SELECT * FROM %s", target, with, source); - - log.info("Running import for %s", target); - long start = System.nanoTime(); - long rows = queryRunner.execute(session, sql).getUpdateCount().getAsLong(); - log.info("Imported %s rows for %s in %s", rows, target, nanosSince(start)); - } - - public static Session createSession() - { - return createSession("tpch"); - } - - public static Session createSession(String schema) - { - SessionPropertyManager sessionPropertyManager = new SessionPropertyManager( - ImmutableSet.of(new SystemSessionProperties()), - CatalogServiceProvider.singleton( - createTestCatalogHandle("raptor"), - Maps.uniqueIndex(new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties(), PropertyMetadata::getName))); - return testSessionBuilder(sessionPropertyManager) - .setCatalog("raptor") - .setSchema(schema) - .setSystemProperty("enable_intermediate_aggregations", "true") - .build(); - } - - public static Path createTempDirectory(String name) - { - try { - Path tempDirectory = Files.createTempDirectory(name); - tempDirectory.toFile().deleteOnExit(); - return tempDirectory.toAbsolutePath(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static void main(String[] args) - throws Exception - { - QueryRunner queryRunner = RaptorQueryRunner.builder() - .addCoordinatorProperty("http-server.http.port", "8080") - .build(); - Logger log = Logger.get(RaptorQueryRunner.class); - log.info("======== SERVER STARTED ========"); - log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketFunction.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketFunction.java deleted file mode 100644 index 1e389c2662fe..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketFunction.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.connector.BucketFunction; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import static io.trino.block.BlockAssertions.createIntsBlock; -import static io.trino.block.BlockAssertions.createLongsBlock; -import static io.trino.block.BlockAssertions.createStringsBlock; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestRaptorBucketFunction -{ - @Test - public void testBigint() - { - BucketFunction function = bucketFunction(50, BIGINT); - assertThat(getBucket(function, createLongsBlock(123456789012L))).isEqualTo(12); - assertThat(getBucket(function, createLongsBlock(454345325))).isEqualTo(16); - assertThat(getBucket(function, createLongsBlock(365363))).isEqualTo(42); - assertThat(getBucket(function, createLongsBlock(45645747))).isEqualTo(41); - assertThat(getBucket(function, createLongsBlock(3244))).isEqualTo(29); - - function = bucketFunction(2, BIGINT); - assertThat(getBucket(function, createLongsBlock(123456789012L))).isEqualTo(0); - assertThat(getBucket(function, createLongsBlock(454345325))).isEqualTo(0); - assertThat(getBucket(function, createLongsBlock(365363))).isEqualTo(0); - assertThat(getBucket(function, createLongsBlock(45645747))).isEqualTo(1); - assertThat(getBucket(function, createLongsBlock(3244))).isEqualTo(1); - } - - @Test - public void testInteger() - { - BucketFunction function = bucketFunction(50, INTEGER); - assertThat(getBucket(function, createIntsBlock(454345325))).isEqualTo(16); - assertThat(getBucket(function, createIntsBlock(365363))).isEqualTo(42); - assertThat(getBucket(function, createIntsBlock(45645747))).isEqualTo(41); - assertThat(getBucket(function, createIntsBlock(3244))).isEqualTo(29); - } - - @Test - public void testVarchar() - { - BucketFunction function = bucketFunction(50, createUnboundedVarcharType()); - assertThat(getBucket(function, createStringsBlock("lorem ipsum"))).isEqualTo(2); - assertThat(getBucket(function, createStringsBlock("lorem"))).isEqualTo(26); - assertThat(getBucket(function, createStringsBlock("ipsum"))).isEqualTo(3); - assertThat(getBucket(function, createStringsBlock("hello"))).isEqualTo(19); - } - - @Test - public void testVarcharBigint() - { - BucketFunction function = bucketFunction(50, createUnboundedVarcharType(), BIGINT); - assertThat(getBucket(function, createStringsBlock("lorem ipsum"), createLongsBlock(123456789012L))).isEqualTo(24); - assertThat(getBucket(function, createStringsBlock("lorem"), createLongsBlock(454345325))).isEqualTo(32); - assertThat(getBucket(function, createStringsBlock("ipsum"), createLongsBlock(365363))).isEqualTo(21); - assertThat(getBucket(function, createStringsBlock("hello"), createLongsBlock(45645747))).isEqualTo(34); - assertThat(getBucket(function, createStringsBlock("world"), createLongsBlock(3244))).isEqualTo(4); - } - - private static int getBucket(BucketFunction function, Block... blocks) - { - return function.getBucket(new Page(blocks), 0); - } - - private static BucketFunction bucketFunction(int bucketCount, Type... types) - { - return new RaptorBucketFunction(bucketCount, ImmutableList.copyOf(types)); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketedConnectorTest.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketedConnectorTest.java deleted file mode 100644 index c1a1ceaff756..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorBucketedConnectorTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.testing.QueryRunner; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class TestRaptorBucketedConnectorTest - extends BaseRaptorConnectorTest -{ - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - return RaptorQueryRunner.builder() - .enableBucketed() - .addConnectorProperty("storage.compaction-enabled", "false") - .setInitialTables(REQUIRED_TPCH_TABLES) - .build(); - } - - @Test - @Override - public void testShowCreateTable() - { - assertThat(computeActual("SHOW CREATE TABLE orders").getOnlyValue()) - .isEqualTo("CREATE TABLE raptor.tpch.orders (\n" + - " orderkey bigint,\n" + - " custkey bigint,\n" + - " orderstatus varchar(1),\n" + - " totalprice double,\n" + - " orderdate date,\n" + - " orderpriority varchar(15),\n" + - " clerk varchar(15),\n" + - " shippriority integer,\n" + - " comment varchar(79)\n" + - ")\n" + - "WITH (\n" + - " bucket_count = 25,\n" + - " bucketed_on = ARRAY['orderkey'],\n" + - " distribution_name = 'order'\n" + - ")"); - } - - @Test - public void testShardsSystemTableBucketNumber() - { - assertQuery("" + - "SELECT count(DISTINCT bucket_number)\n" + - "FROM system.shards\n" + - "WHERE table_schema = 'tpch'\n" + - " AND table_name = 'orders'", - "SELECT 25"); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java deleted file mode 100644 index aeb493179200..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.slice.Slice; -import io.trino.operator.PagesIndex; -import io.trino.operator.PagesIndexPageSorter; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.NodeManager; -import io.trino.spi.Page; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.SqlDate; -import io.trino.spi.type.SqlTimestamp; -import io.trino.spi.type.Type; -import io.trino.testing.MaterializedResult; -import io.trino.testing.TestingConnectorSession; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.nio.file.Files; -import java.util.Collection; -import java.util.Optional; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.operator.scalar.timestamp.VarcharToTimestampCast.castToShortTimestamp; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.TestRaptorStorageManager.createRaptorStorageManager; -import static io.trino.spi.connector.RetryMode.NO_RETRIES; -import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.testing.TestingPageSinkId.TESTING_PAGE_SINK_ID; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static io.trino.util.DateTimeUtils.parseDate; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestRaptorConnector -{ - private static final ConnectorSession SESSION = TestingConnectorSession.builder() - .setPropertyMetadata(new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties()) - .build(); - - private Handle dummyHandle; - private MetadataDao metadataDao; - private File dataDir; - private RaptorConnector connector; - - @BeforeEach - public void setup() - throws Exception - { - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - metadataDao = dbi.onDemand(MetadataDao.class); - createTablesWithRetry(dbi); - dataDir = Files.createTempDirectory(null).toFile(); - - CatalogName connectorId = new CatalogName("test"); - NodeManager nodeManager = new TestingNodeManager(); - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - ShardManager shardManager = createShardManager(dbi); - StorageManager storageManager = createRaptorStorageManager(dbi, dataDir); - StorageManagerConfig config = new StorageManagerConfig(); - connector = new RaptorConnector( - new LifeCycleManager(ImmutableList.of(), null), - new TestingNodeManager(), - new RaptorMetadataFactory(dbi, shardManager), - new RaptorSplitManager(connectorId, nodeSupplier, shardManager, false), - new RaptorPageSourceProvider(storageManager), - new RaptorPageSinkProvider(storageManager, - new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)), - config), - new RaptorNodePartitioningProvider(nodeSupplier), - new RaptorSessionProperties(config), - new RaptorTableProperties(TESTING_TYPE_MANAGER), - ImmutableSet.of(), - Optional.empty(), - dbi); - } - - @AfterEach - public void tearDown() - throws Exception - { - dummyHandle.close(); - dummyHandle = null; - deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); - } - - @Test - public void testMaintenanceBlocked() - { - long tableId1 = createTable("test1"); - long tableId2 = createTable("test2"); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isFalse(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isFalse(); - - // begin delete for table1 - ConnectorTransactionHandle txn1 = beginTransaction(); - ConnectorTableHandle handle1 = getTableHandle(connector.getMetadata(SESSION, txn1), "test1"); - connector.getMetadata(SESSION, txn1).beginMerge(SESSION, handle1, NO_RETRIES); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isTrue(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isFalse(); - - // begin delete for table2 - ConnectorTransactionHandle txn2 = beginTransaction(); - ConnectorTableHandle handle2 = getTableHandle(connector.getMetadata(SESSION, txn2), "test2"); - connector.getMetadata(SESSION, txn2).beginMerge(SESSION, handle2, NO_RETRIES); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isTrue(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isTrue(); - - // begin another delete for table1 - ConnectorTransactionHandle txn3 = beginTransaction(); - ConnectorTableHandle handle3 = getTableHandle(connector.getMetadata(SESSION, txn3), "test1"); - connector.getMetadata(SESSION, txn3).beginMerge(SESSION, handle3, NO_RETRIES); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isTrue(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isTrue(); - - // commit first delete for table1 - connector.commit(txn1); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isTrue(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isTrue(); - - // rollback second delete for table1 - connector.rollback(txn3); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isFalse(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isTrue(); - - // commit delete for table2 - connector.commit(txn2); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId1)).isFalse(); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId2)).isFalse(); - } - - @Test - public void testMaintenanceUnblockedOnStart() - { - long tableId = createTable("test"); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId)).isFalse(); - metadataDao.blockMaintenance(tableId); - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId)).isTrue(); - - connector.start(); - - assertThat(metadataDao.isMaintenanceBlockedLocked(tableId)).isFalse(); - } - - @Test - public void testTemporalShardSplit() - throws Exception - { - // Same date should be in same split - assertSplitShard(DATE, "2001-08-22", "2001-08-22", 1); - - // Same date should be in different splits - assertSplitShard(DATE, "2001-08-22", "2001-08-23", 2); - - // Same timestamp should be in same split - assertSplitShard(TIMESTAMP_MILLIS, "2001-08-22 00:00:01.000", "2001-08-22 23:59:01.000", 1); - - // Same timestamp should be in different splits - assertSplitShard(TIMESTAMP_MILLIS, "2001-08-22 23:59:01.000", "2001-08-23 00:00:01.000", 2); - } - - private void assertSplitShard(Type temporalType, String min, String max, int expectedSplits) - throws Exception - { - ConnectorSession session = TestingConnectorSession.builder() - .setPropertyMetadata(new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties()) - .build(); - - ConnectorTransactionHandle transaction = beginTransaction(); - connector.getMetadata(SESSION, transaction).createTable( - SESSION, - new ConnectorTableMetadata( - new SchemaTableName("test", "test"), - ImmutableList.of(new ColumnMetadata("id", BIGINT), new ColumnMetadata("time", temporalType)), - ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "time")), - SaveMode.FAIL); - connector.commit(transaction); - - ConnectorTransactionHandle txn1 = beginTransaction(); - ConnectorTableHandle handle1 = getTableHandle(connector.getMetadata(SESSION, txn1), "test"); - ConnectorInsertTableHandle insertTableHandle = connector.getMetadata(SESSION, txn1).beginInsert(session, handle1, ImmutableList.of(), NO_RETRIES); - ConnectorPageSink raptorPageSink = connector.getPageSinkProvider().createPageSink(txn1, session, insertTableHandle, TESTING_PAGE_SINK_ID); - - Object timestamp1 = null; - Object timestamp2 = null; - if (temporalType.equals(TIMESTAMP_MILLIS)) { - timestamp1 = SqlTimestamp.newInstance(3, castToShortTimestamp(TIMESTAMP_MILLIS.getPrecision(), min), 0); - timestamp2 = SqlTimestamp.newInstance(3, castToShortTimestamp(TIMESTAMP_MILLIS.getPrecision(), max), 0); - } - else if (temporalType.equals(DATE)) { - timestamp1 = new SqlDate(parseDate(min)); - timestamp2 = new SqlDate(parseDate(max)); - } - - Page inputPage = MaterializedResult.resultBuilder(session, ImmutableList.of(BIGINT, temporalType)) - .row(1L, timestamp1) - .row(2L, timestamp2) - .build() - .toPage(); - - raptorPageSink.appendPage(inputPage); - - Collection shards = raptorPageSink.finish().get(); - assertThat(shards.size()).isEqualTo(expectedSplits); - connector.getMetadata(session, txn1).dropTable(session, handle1); - connector.commit(txn1); - } - - private long createTable(String name) - { - ConnectorTransactionHandle transaction = beginTransaction(); - connector.getMetadata(SESSION, transaction).createTable( - SESSION, - new ConnectorTableMetadata( - new SchemaTableName("test", name), - ImmutableList.of(new ColumnMetadata("id", BIGINT))), - SaveMode.FAIL); - connector.commit(transaction); - - transaction = beginTransaction(); - ConnectorTableHandle tableHandle = getTableHandle(connector.getMetadata(SESSION, transaction), name); - connector.commit(transaction); - return ((RaptorTableHandle) tableHandle).getTableId(); - } - - private ConnectorTransactionHandle beginTransaction() - { - return connector.beginTransaction(READ_COMMITTED, false, true); - } - - private static ConnectorTableHandle getTableHandle(ConnectorMetadata metadata, String name) - { - return metadata.getTableHandle(SESSION, new SchemaTableName("test", name), Optional.empty(), Optional.empty()); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnectorTest.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnectorTest.java deleted file mode 100644 index acdb1bd522c3..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnectorTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import io.trino.testing.QueryRunner; - -public class TestRaptorConnectorTest - extends BaseRaptorConnectorTest -{ - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - return RaptorQueryRunner.builder() - .addConnectorProperty("storage.compaction-enabled", "false") - .setInitialTables(REQUIRED_TPCH_TABLES) - .build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorMySqlConnectorTest.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorMySqlConnectorTest.java deleted file mode 100644 index 1c322d1515aa..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorMySqlConnectorTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableMap; -import io.trino.plugin.tpch.TpchPlugin; -import io.trino.testing.DistributedQueryRunner; -import io.trino.testing.QueryRunner; -import org.junit.jupiter.api.AfterAll; -import org.testcontainers.containers.MySQLContainer; - -import java.util.Map; - -import static io.trino.plugin.raptor.legacy.RaptorQueryRunner.copyTables; -import static io.trino.plugin.raptor.legacy.RaptorQueryRunner.createSession; -import static io.trino.plugin.raptor.legacy.RaptorQueryRunner.createTempDirectory; -import static java.lang.String.format; - -public class TestRaptorMySqlConnectorTest - extends BaseRaptorConnectorTest -{ - private MySQLContainer mysqlContainer; - - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - mysqlContainer = new MySQLContainer<>("mysql:8.0.36"); - mysqlContainer.start(); - return createRaptorMySqlQueryRunner(getJdbcUrl(mysqlContainer)); - } - - @AfterAll - public final void destroy() - { - mysqlContainer.close(); - mysqlContainer = null; - } - - private static String getJdbcUrl(MySQLContainer container) - { - return format("%s?user=%s&password=%s&useSSL=false&allowPublicKeyRetrieval=true", - container.getJdbcUrl(), - container.getUsername(), - container.getPassword()); - } - - private static QueryRunner createRaptorMySqlQueryRunner(String mysqlUrl) - throws Exception - { - QueryRunner queryRunner = DistributedQueryRunner.builder(createSession("tpch")).build(); - - queryRunner.installPlugin(new TpchPlugin()); - queryRunner.createCatalog("tpch", "tpch"); - - queryRunner.installPlugin(new RaptorPlugin()); - Map raptorProperties = ImmutableMap.builder() - .put("metadata.db.type", "mysql") - .put("metadata.db.url", mysqlUrl) - .put("storage.compaction-enabled", "false") - .put("storage.data-directory", createTempDirectory("raptor-db").toString()) - .put("storage.max-shard-rows", "2000") - .put("backup.provider", "file") - .put("backup.directory", createTempDirectory("raptor-db").toString()) - .buildOrThrow(); - - queryRunner.createCatalog("raptor", "raptor_legacy", raptorProperties); - - copyTables(queryRunner, "tpch", createSession(), false, REQUIRED_TPCH_TABLES); - - return queryRunner; - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorPlugin.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorPlugin.java deleted file mode 100644 index b713617f8627..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorPlugin.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy; - -import com.google.common.collect.ImmutableMap; -import io.trino.spi.Plugin; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.testing.TestingConnectorContext; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.nio.file.Files; -import java.util.Map; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.testing.Assertions.assertInstanceOf; - -public class TestRaptorPlugin -{ - @Test - public void testPlugin() - throws Exception - { - Plugin plugin = new RaptorPlugin(); - ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); - assertInstanceOf(factory, RaptorConnectorFactory.class); - - File tmpDir = Files.createTempDirectory(null).toFile(); - try { - Map config = ImmutableMap.builder() - .put("metadata.db.type", "h2") - .put("metadata.db.filename", tmpDir.getAbsolutePath()) - .put("storage.data-directory", tmpDir.getAbsolutePath()) - .put("bootstrap.quiet", "true") - .buildOrThrow(); - - factory.create("test", config, new TestingConnectorContext()).shutdown(); - } - finally { - deleteRecursively(tmpDir.toPath(), ALLOW_INSECURE); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/AbstractTestBackupStore.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/AbstractTestBackupStore.java deleted file mode 100644 index 948d4ee61ea7..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/AbstractTestBackupStore.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.nio.file.Path; -import java.util.UUID; - -import static java.nio.file.Files.readAllBytes; -import static java.nio.file.Files.writeString; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThat; - -public abstract class AbstractTestBackupStore -{ - protected Path temporary; - protected T store; - - @Test - public void testBackupStore() - throws Exception - { - // backup first file - File file1 = temporary.resolve("file1").toFile(); - writeString(file1.toPath(), "hello world"); - UUID uuid1 = randomUUID(); - - assertThat(store.shardExists(uuid1)).isFalse(); - store.backupShard(uuid1, file1); - assertThat(store.shardExists(uuid1)).isTrue(); - - // backup second file - File file2 = temporary.resolve("file2").toFile(); - writeString(file2.toPath(), "bye bye"); - UUID uuid2 = randomUUID(); - - assertThat(store.shardExists(uuid2)).isFalse(); - store.backupShard(uuid2, file2); - assertThat(store.shardExists(uuid2)).isTrue(); - - // verify first file - File restore1 = temporary.resolve("restore1").toFile(); - store.restoreShard(uuid1, restore1); - assertThat(readAllBytes(file1.toPath())).isEqualTo(readAllBytes(restore1.toPath())); - - // verify second file - File restore2 = temporary.resolve("restore2").toFile(); - store.restoreShard(uuid2, restore2); - assertThat(readAllBytes(file2.toPath())).isEqualTo(readAllBytes(restore2.toPath())); - - // verify random UUID does not exist - assertThat(store.shardExists(randomUUID())).isFalse(); - - // delete first file - assertThat(store.shardExists(uuid1)).isTrue(); - assertThat(store.shardExists(uuid2)).isTrue(); - - store.deleteShard(uuid1); - store.deleteShard(uuid1); - - assertThat(store.shardExists(uuid1)).isFalse(); - assertThat(store.shardExists(uuid2)).isTrue(); - - // delete random UUID - store.deleteShard(randomUUID()); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupConfig.java deleted file mode 100644 index db9ce39cdabf..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupConfig.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class TestBackupConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(BackupConfig.class) - .setProvider(null) - .setTimeoutThreads(1000) - .setTimeout(new Duration(1, MINUTES)) - .setBackupThreads(5)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("backup.provider", "file") - .put("backup.timeout", "42s") - .put("backup.timeout-threads", "13") - .put("backup.threads", "3") - .buildOrThrow(); - - BackupConfig expected = new BackupConfig() - .setProvider("file") - .setTimeout(new Duration(42, SECONDS)) - .setTimeoutThreads(13) - .setBackupThreads(3); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupManager.java deleted file mode 100644 index a93596ab649f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestBackupManager.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import io.trino.plugin.raptor.legacy.storage.BackupStats; -import io.trino.plugin.raptor.legacy.storage.FileStorageService; -import io.trino.spi.TrinoException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ThreadLocalRandom; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_ERROR; -import static java.nio.file.Files.createTempDirectory; -import static java.nio.file.Files.writeString; -import static java.util.Objects.requireNonNull; -import static java.util.UUID.randomUUID; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestBackupManager -{ - private static final UUID FAILURE_UUID = randomUUID(); - private static final UUID CORRUPTION_UUID = randomUUID(); - - private Path temporary; - private BackupStore backupStore; - private FileStorageService storageService; - private BackupManager backupManager; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - - FileBackupStore fileStore = new FileBackupStore(temporary.resolve("backup").toFile()); - fileStore.start(); - backupStore = new TestingBackupStore(fileStore); - - storageService = new FileStorageService(temporary.resolve("data").toFile()); - storageService.start(); - - backupManager = new BackupManager(Optional.of(backupStore), storageService, 5); - } - - @AfterEach - public void tearDown() - throws Exception - { - deleteRecursively(temporary, ALLOW_INSECURE); - backupManager.shutdown(); - } - - @Test - public void testSimple() - throws Exception - { - assertEmptyStagingDirectory(); - assertBackupStats(0, 0, 0); - - List> futures = new ArrayList<>(); - List uuids = new ArrayList<>(5); - for (int i = 0; i < 5; i++) { - File file = temporary.resolve("file" + i).toFile(); - writeString(file.toPath(), "hello world"); - uuids.add(randomUUID()); - - futures.add(backupManager.submit(uuids.get(i), file)); - } - futures.forEach(CompletableFuture::join); - for (UUID uuid : uuids) { - assertThat(backupStore.shardExists(uuid)).isTrue(); - } - - assertBackupStats(5, 0, 0); - assertEmptyStagingDirectory(); - } - - @Test - public void testFailure() - throws Exception - { - assertEmptyStagingDirectory(); - assertBackupStats(0, 0, 0); - - File file = temporary.resolve("failure").toFile(); - writeString(file.toPath(), "hello world"); - - assertThatThrownBy(() -> backupManager.submit(FAILURE_UUID, file).get(10, SECONDS)) - .isInstanceOfSatisfying(ExecutionException.class, wrapper -> { - TrinoException e = (TrinoException) wrapper.getCause(); - assertThat(e.getErrorCode()).isEqualTo(RAPTOR_BACKUP_ERROR.toErrorCode()); - assertThat(e.getMessage()).isEqualTo("Backup failed for testing"); - }); - - assertBackupStats(0, 1, 0); - assertEmptyStagingDirectory(); - } - - @Test - public void testCorruption() - throws Exception - { - assertEmptyStagingDirectory(); - assertBackupStats(0, 0, 0); - - File file = temporary.resolve("corrupt").toFile(); - writeString(file.toPath(), "hello world"); - - assertThatThrownBy(() -> backupManager.submit(CORRUPTION_UUID, file).get(10, SECONDS)) - .isInstanceOfSatisfying(ExecutionException.class, wrapper -> { - TrinoException e = (TrinoException) wrapper.getCause(); - assertThat(e.getErrorCode()).isEqualTo(RAPTOR_BACKUP_CORRUPTION.toErrorCode()); - assertThat(e.getMessage()).isEqualTo("Backup is corrupt after write: " + CORRUPTION_UUID); - }); - - File quarantineBase = storageService.getQuarantineFile(CORRUPTION_UUID); - assertThat(new File(quarantineBase.getPath() + ".original")).isFile(); - assertThat(new File(quarantineBase.getPath() + ".restored")).isFile(); - - assertBackupStats(0, 1, 1); - assertEmptyStagingDirectory(); - } - - private void assertEmptyStagingDirectory() - { - File staging = storageService.getStagingFile(randomUUID()).getParentFile(); - assertThat(staging.list()).isEqualTo(new String[] {}); - } - - private void assertBackupStats(int successCount, int failureCount, int corruptionCount) - { - BackupStats stats = backupManager.getStats(); - assertThat(stats.getBackupSuccess().getTotalCount()).isEqualTo(successCount); - assertThat(stats.getBackupFailure().getTotalCount()).isEqualTo(failureCount); - assertThat(stats.getBackupCorruption().getTotalCount()).isEqualTo(corruptionCount); - } - - private static class TestingBackupStore - implements BackupStore - { - private final BackupStore delegate; - - private TestingBackupStore(BackupStore delegate) - { - this.delegate = requireNonNull(delegate, "delegate is null"); - } - - @Override - public void backupShard(UUID uuid, File source) - { - if (uuid.equals(FAILURE_UUID)) { - throw new TrinoException(RAPTOR_BACKUP_ERROR, "Backup failed for testing"); - } - delegate.backupShard(uuid, source); - } - - @Override - public void restoreShard(UUID uuid, File target) - { - delegate.restoreShard(uuid, target); - if (uuid.equals(CORRUPTION_UUID)) { - corruptFile(target); - } - } - - @Override - public boolean deleteShard(UUID uuid) - { - return delegate.deleteShard(uuid); - } - - @Override - public boolean shardExists(UUID uuid) - { - return delegate.shardExists(uuid); - } - - private static void corruptFile(File path) - { - // flip a bit at a random offset - try (RandomAccessFile file = new RandomAccessFile(path, "rw")) { - if (file.length() == 0) { - throw new RuntimeException("file is empty"); - } - long offset = ThreadLocalRandom.current().nextLong(file.length()); - file.seek(offset); - int value = file.read() ^ 0x01; - file.seek(offset); - file.write(value); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupConfig.java deleted file mode 100644 index a08e89f8529f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestFileBackupConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(FileBackupConfig.class) - .setBackupDirectory(null)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("backup.directory", "/backup"); - - FileBackupConfig expected = new FileBackupConfig() - .setBackupDirectory(new File("/backup")); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupStore.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupStore.java deleted file mode 100644 index 6b0060f75160..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestFileBackupStore.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; - -import java.io.File; -import java.io.IOException; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; - -@TestInstance(PER_CLASS) -public class TestFileBackupStore - extends AbstractTestBackupStore -{ - @BeforeAll - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - store = new FileBackupStore(temporary.resolve("backup").toFile()); - store.start(); - } - - @AfterAll - public void tearDown() - throws Exception - { - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testFilePaths() - { - UUID uuid = UUID.fromString("701e1a79-74f7-4f56-b438-b41e8e7d019d"); - File expected = temporary.resolve("backup").resolve("70").resolve("1e").resolve(format("%s.orc", uuid)).toFile(); - assertThat(store.getBackupFile(uuid)).isEqualTo(expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupConfig.java deleted file mode 100644 index db650c9b807f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.net.URI; -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestHttpBackupConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(HttpBackupConfig.class) - .setUri(null)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("backup.http.uri", "http://example.net:8080"); - - HttpBackupConfig expected = new HttpBackupConfig() - .setUri(URI.create("http://example.net:8080")); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupStore.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupStore.java deleted file mode 100644 index 975b86c3bf6d..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestHttpBackupStore.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.common.collect.ImmutableMap; -import com.google.inject.Binder; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.http.server.HttpServerInfo; -import io.airlift.http.server.testing.TestingHttpServerModule; -import io.airlift.jaxrs.JaxrsModule; -import io.airlift.json.JsonModule; -import io.airlift.node.testing.TestingNodeModule; -import io.trino.spi.NodeManager; -import io.trino.testing.TestingNodeManager; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; - -import java.io.IOException; -import java.net.URI; -import java.util.Map; -import java.util.function.Supplier; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static com.google.inject.util.Modules.override; -import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; -import static java.nio.file.Files.createTempDirectory; - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -@Execution(ExecutionMode.SAME_THREAD) -public class TestHttpBackupStore - extends AbstractTestBackupStore -{ - private LifeCycleManager lifeCycleManager; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - - Map properties = ImmutableMap.of("backup.http.uri", "http://localhost:8080"); - - Bootstrap app = new Bootstrap( - new TestingNodeModule(), - new TestingHttpServerModule(), - new JsonModule(), - new JaxrsModule(), - binder -> jaxrsBinder(binder).bind(TestingHttpBackupResource.class), - binder -> binder.bind(NodeManager.class).toInstance(new TestingNodeManager()), - override(new HttpBackupModule()).with(new TestingModule())); - - Injector injector = app - .setRequiredConfigurationProperties(properties) - .doNotInitializeLogging() - .quiet() - .initialize(); - - lifeCycleManager = injector.getInstance(LifeCycleManager.class); - - store = injector.getInstance(BackupStore.class); - } - - @AfterEach - public void teardown() - throws IOException - { - deleteRecursively(temporary, ALLOW_INSECURE); - if (lifeCycleManager != null) { - lifeCycleManager.stop(); - } - } - - private static class TestingModule - implements Module - { - @Override - public void configure(Binder binder) {} - - @Provides - @Singleton - @ForHttpBackup - public Supplier createBackupUriSupplier(HttpServerInfo serverInfo) - { - return serverInfo::getHttpUri; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestingHttpBackupResource.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestingHttpBackupResource.java deleted file mode 100644 index 0d6a237f7f8b..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/backup/TestingHttpBackupResource.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.backup; - -import com.google.errorprone.annotations.concurrent.GuardedBy; -import com.google.inject.Inject; -import io.airlift.slice.Slices; -import io.airlift.slice.XxHash64; -import io.trino.server.GoneException; -import io.trino.spi.NodeManager; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.ForbiddenException; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.HEAD; -import jakarta.ws.rs.HeaderParam; -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.PUT; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.Context; -import jakarta.ws.rs.core.Response; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import static io.trino.plugin.raptor.legacy.backup.HttpBackupStore.CONTENT_XXH64; -import static io.trino.plugin.raptor.legacy.backup.HttpBackupStore.TRINO_ENVIRONMENT; -import static jakarta.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM; -import static java.lang.Long.parseUnsignedLong; -import static java.util.Objects.requireNonNull; - -@Path("/") -public class TestingHttpBackupResource -{ - private final String environment; - - @GuardedBy("this") - private final Map shards = new HashMap<>(); - - @Inject - public TestingHttpBackupResource(NodeManager nodeManager) - { - this(nodeManager.getEnvironment()); - } - - public TestingHttpBackupResource(String environment) - { - this.environment = requireNonNull(environment, "environment is null"); - } - - @HEAD - @Path("{uuid}") - public synchronized Response headRequest( - @HeaderParam(TRINO_ENVIRONMENT) String environment, - @PathParam("uuid") UUID uuid) - { - checkEnvironment(environment); - if (!shards.containsKey(uuid)) { - throw new NotFoundException(); - } - if (shards.get(uuid) == null) { - throw new GoneException(); - } - return Response.noContent().build(); - } - - @GET - @Path("{uuid}") - @Produces(APPLICATION_OCTET_STREAM) - public synchronized Response getRequest( - @HeaderParam(TRINO_ENVIRONMENT) String environment, - @PathParam("uuid") UUID uuid) - { - checkEnvironment(environment); - if (!shards.containsKey(uuid)) { - throw new NotFoundException(); - } - byte[] bytes = shards.get(uuid); - if (bytes == null) { - throw new GoneException(); - } - return Response.ok(bytes).build(); - } - - @PUT - @Path("{uuid}") - public synchronized Response putRequest( - @HeaderParam(TRINO_ENVIRONMENT) String environment, - @HeaderParam(CONTENT_XXH64) String hexHash, - @Context HttpServletRequest request, - @PathParam("uuid") UUID uuid, - byte[] bytes) - { - checkEnvironment(environment); - if ((request.getContentLength() < 0) || (bytes.length != request.getContentLength())) { - throw new BadRequestException(); - } - if (parseUnsignedLong(hexHash, 16) != XxHash64.hash(Slices.wrappedBuffer(bytes))) { - throw new BadRequestException(); - } - if (shards.containsKey(uuid)) { - byte[] existing = shards.get(uuid); - if ((existing == null) || !Arrays.equals(bytes, existing)) { - throw new ForbiddenException(); - } - } - shards.put(uuid, bytes); - return Response.noContent().build(); - } - - @DELETE - @Path("{uuid}") - public synchronized Response deleteRequest( - @HeaderParam(TRINO_ENVIRONMENT) String environment, - @PathParam("uuid") UUID uuid) - { - checkEnvironment(environment); - if (!shards.containsKey(uuid)) { - throw new NotFoundException(); - } - if (shards.get(uuid) == null) { - throw new GoneException(); - } - shards.put(uuid, null); - return Response.noContent().build(); - } - - private void checkEnvironment(String environment) - { - if (!this.environment.equals(environment)) { - throw new ForbiddenException(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/ShardNode.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/ShardNode.java deleted file mode 100644 index 3087dd72fdc2..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/ShardNode.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import java.util.Objects; -import java.util.UUID; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class ShardNode -{ - private final UUID shardUuid; - private final String nodeIdentifier; - - public ShardNode(UUID shardUuid, String nodeIdentifier) - { - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null"); - } - - public UUID getShardUuid() - { - return shardUuid; - } - - public String getNodeIdentifier() - { - return nodeIdentifier; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - ShardNode other = (ShardNode) obj; - return Objects.equals(this.shardUuid, other.shardUuid) && - Objects.equals(this.nodeIdentifier, other.nodeIdentifier); - } - - @Override - public int hashCode() - { - return Objects.hash(shardUuid, nodeIdentifier); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("shardUuid", shardUuid) - .add("nodeIdentifier", nodeIdentifier) - .toString(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestAssignmentLimiter.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestAssignmentLimiter.java deleted file mode 100644 index f319ddf6184a..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestAssignmentLimiter.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableSet; -import io.airlift.testing.TestingTicker; -import io.airlift.units.Duration; -import io.trino.spi.ErrorCodeSupplier; -import org.junit.jupiter.api.Test; - -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_REASSIGNMENT_DELAY; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_REASSIGNMENT_THROTTLE; -import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; -import static java.util.concurrent.TimeUnit.MINUTES; - -public class TestAssignmentLimiter -{ - @Test - public void testLimiter() - { - TestingTicker ticker = new TestingTicker(); - AssignmentLimiter limiter = new AssignmentLimiter( - ImmutableSet::of, - ticker, - new Duration(5, MINUTES), - new Duration(10, MINUTES)); - - // 4:00 - assertCheckFails(limiter, "A", RAPTOR_REASSIGNMENT_DELAY); - - // 4:01 - ticker.increment(1, MINUTES); - assertCheckFails(limiter, "A", RAPTOR_REASSIGNMENT_DELAY); - assertCheckFails(limiter, "B", RAPTOR_REASSIGNMENT_DELAY); - - // 4:05 - ticker.increment(4, MINUTES); - limiter.checkAssignFrom("A"); - assertCheckFails(limiter, "B", RAPTOR_REASSIGNMENT_DELAY); - - // 4:06 - ticker.increment(1, MINUTES); - assertCheckFails(limiter, "B", RAPTOR_REASSIGNMENT_THROTTLE); - assertCheckFails(limiter, "C", RAPTOR_REASSIGNMENT_DELAY); - - // 4:14 - ticker.increment(8, MINUTES); - assertCheckFails(limiter, "B", RAPTOR_REASSIGNMENT_THROTTLE); - assertCheckFails(limiter, "C", RAPTOR_REASSIGNMENT_THROTTLE); - - // 4:15 - ticker.increment(1, MINUTES); - limiter.checkAssignFrom("B"); - assertCheckFails(limiter, "C", RAPTOR_REASSIGNMENT_THROTTLE); - - // 4:24 - ticker.increment(9, MINUTES); - assertCheckFails(limiter, "C", RAPTOR_REASSIGNMENT_THROTTLE); - - // 4:25 - ticker.increment(1, MINUTES); - limiter.checkAssignFrom("A"); - - // 4:30 - ticker.increment(5, MINUTES); - limiter.checkAssignFrom("A"); - limiter.checkAssignFrom("B"); - limiter.checkAssignFrom("C"); - } - - private static void assertCheckFails(AssignmentLimiter limiter, String node, ErrorCodeSupplier expected) - { - assertTrinoExceptionThrownBy(() -> limiter.checkAssignFrom(node)) - .hasErrorCode(expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseConfig.java deleted file mode 100644 index b9f8daa3e5ff..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestDatabaseConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(DatabaseConfig.class) - .setDatabaseType(null)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("metadata.db.type", "h2"); - - DatabaseConfig expected = new DatabaseConfig() - .setDatabaseType("h2"); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseShardManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseShardManager.java deleted file mode 100644 index 3e4c7341886a..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestDatabaseShardManager.java +++ /dev/null @@ -1,818 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.base.Ticker; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; -import io.airlift.slice.Slice; -import io.airlift.testing.TestingTicker; -import io.airlift.units.Duration; -import io.trino.client.NodeVersion; -import io.trino.metadata.InternalNode; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.Node; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.predicate.ValueSet; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.result.ResultIterator; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.LocalDate; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.base.Ticker.systemTicker; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.collect.Iterators.concat; -import static com.google.common.collect.Iterators.transform; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.storage.ShardStats.MAX_BINARY_INDEX_SIZE; -import static io.trino.spi.StandardErrorCode.SERVER_STARTING_UP; -import static io.trino.spi.StandardErrorCode.TRANSACTION_CONFLICT; -import static io.trino.spi.predicate.Range.greaterThan; -import static io.trino.spi.predicate.Range.greaterThanOrEqual; -import static io.trino.spi.predicate.Range.lessThan; -import static io.trino.spi.predicate.Range.range; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; -import static java.lang.String.format; -import static java.time.ZoneOffset.UTC; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestDatabaseShardManager -{ - private Jdbi dbi; - private Handle dummyHandle; - private File dataDir; - private ShardManager shardManager; - - @BeforeEach - public void setup() - throws Exception - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - dataDir = Files.createTempDirectory(null).toFile(); - shardManager = createShardManager(dbi); - } - - @AfterEach - public void teardown() - throws IOException - { - dummyHandle.close(); - dummyHandle = null; - deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); - } - - @Test - public void testCommit() - { - long tableId = createTable("test"); - - List shards = ImmutableList.builder() - .add(shardInfo(UUID.randomUUID(), "node1")) - .add(shardInfo(UUID.randomUUID(), "node1")) - .add(shardInfo(UUID.randomUUID(), "node2")) - .build(); - - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - Set actual = getShardNodes(tableId, TupleDomain.all()); - assertThat(actual).isEqualTo(toShardNodes(shards)); - } - - @Test - public void testRollback() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - List shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.rollbackTransaction(transactionId); - - assertTrinoExceptionThrownBy(() -> shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0)) - .hasErrorCode(TRANSACTION_CONFLICT) - .hasMessage("Transaction commit failed. Please retry the operation."); - } - - @Test - public void testAssignShard() - { - long tableId = createTable("test"); - UUID shard = UUID.randomUUID(); - List shardNodes = ImmutableList.of(shardInfo(shard, "node1")); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); - - ShardNodes actual = getOnlyElement(getShardNodes(tableId, TupleDomain.all())); - assertThat(actual).isEqualTo(new ShardNodes(shard, ImmutableSet.of("node1"))); - - assertTrinoExceptionThrownBy(() -> shardManager.replaceShardAssignment(tableId, shard, "node2", true)) - .hasErrorCode(SERVER_STARTING_UP) - .hasMessage("Cannot reassign shards while server is starting"); - - // replace shard assignment to another node - shardManager.replaceShardAssignment(tableId, shard, "node2", false); - - actual = getOnlyElement(getShardNodes(tableId, TupleDomain.all())); - assertThat(actual).isEqualTo(new ShardNodes(shard, ImmutableSet.of("node2"))); - - // replacing shard assignment should be idempotent - shardManager.replaceShardAssignment(tableId, shard, "node2", false); - - actual = getOnlyElement(getShardNodes(tableId, TupleDomain.all())); - assertThat(actual).isEqualTo(new ShardNodes(shard, ImmutableSet.of("node2"))); - } - - @Test - public void testGetNodeBytes() - { - long tableId = createTable("test"); - OptionalInt bucketNumber = OptionalInt.empty(); - - UUID shard1 = UUID.randomUUID(); - UUID shard2 = UUID.randomUUID(); - List shardNodes = ImmutableList.of( - new ShardInfo(shard1, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 3, 33, 333, 0), - new ShardInfo(shard2, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 5, 55, 555, 0)); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); - - assertThat(getShardNodes(tableId, TupleDomain.all())).isEqualTo(ImmutableSet.of( - new ShardNodes(shard1, ImmutableSet.of("node1")), - new ShardNodes(shard2, ImmutableSet.of("node1")))); - - assertThat(shardManager.getNodeBytes()).isEqualTo(ImmutableMap.of("node1", 88L)); - - shardManager.replaceShardAssignment(tableId, shard1, "node2", false); - - assertThat(getShardNodes(tableId, TupleDomain.all())).isEqualTo(ImmutableSet.of( - new ShardNodes(shard1, ImmutableSet.of("node2")), - new ShardNodes(shard2, ImmutableSet.of("node1")))); - - assertThat(shardManager.getNodeBytes()).isEqualTo(ImmutableMap.of("node1", 55L, "node2", 33L)); - } - - @Test - public void testGetNodeTableShards() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - List nodes = ImmutableList.of("node1", "node2", "node3"); - - ImmutableList.Builder inputShards = ImmutableList.builder(); - Multimap nodeShardMap = HashMultimap.create(); - for (String node : nodes) { - UUID uuid = UUID.randomUUID(); - nodeShardMap.put(node, uuid); - inputShards.add(shardInfo(uuid, node)); - } - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, inputShards.build(), Optional.empty(), 0); - - for (String node : nodes) { - Set shardMetadata = shardManager.getNodeShards(node); - Set expectedUuids = ImmutableSet.copyOf(nodeShardMap.get(node)); - Set actualUuids = shardMetadata.stream().map(ShardMetadata::getShardUuid).collect(toSet()); - assertThat(actualUuids).isEqualTo(expectedUuids); - } - } - - @Test - public void testGetExistingShards() - { - long tableId = createTable("test"); - UUID shard1 = UUID.randomUUID(); - UUID shard2 = UUID.randomUUID(); - List shardNodes = ImmutableList.of(shardInfo(shard1, "node1"), shardInfo(shard2, "node1")); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); - Set actual = shardManager.getExistingShardUuids(tableId, ImmutableSet.of(shard1, shard2, UUID.randomUUID())); - Set expected = ImmutableSet.of(shard1, shard2); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testReplaceShardUuids() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - List nodes = ImmutableList.of("node1", "node2", "node3"); - List originalUuids = ImmutableList.of(UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID()); - - List oldShards = ImmutableList.builder() - .add(shardInfo(originalUuids.get(0), nodes.get(0))) - .add(shardInfo(originalUuids.get(1), nodes.get(1))) - .add(shardInfo(originalUuids.get(2), nodes.get(2))) - .build(); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, oldShards, Optional.empty(), 0); - - List expectedUuids = ImmutableList.of(UUID.randomUUID(), UUID.randomUUID()); - List newShards = ImmutableList.builder() - .add(shardInfo(expectedUuids.get(0), nodes.get(0))) - .add(shardInfo(expectedUuids.get(1), nodes.get(0))) - .build(); - - Set shardMetadata = shardManager.getNodeShards(nodes.get(0)); - Set replacedUuids = shardMetadata.stream().map(ShardMetadata::getShardUuid).collect(toSet()); - - transactionId = shardManager.beginTransaction(); - shardManager.replaceShardUuids(transactionId, tableId, columns, replacedUuids, newShards, OptionalLong.of(0)); - - shardMetadata = shardManager.getNodeShards(nodes.get(0)); - Set actualUuids = shardMetadata.stream().map(ShardMetadata::getShardUuid).collect(toSet()); - assertThat(actualUuids).isEqualTo(ImmutableSet.copyOf(expectedUuids)); - - // Compute expected all uuids for this table - Set expectedAllUuids = new HashSet<>(originalUuids); - expectedAllUuids.removeAll(replacedUuids); - expectedAllUuids.addAll(expectedUuids); - - // check that shards are replaced in index table as well - Set shardNodes = ImmutableSet.copyOf(shardManager.getShardNodes(tableId, TupleDomain.all())); - Set actualAllUuids = shardNodes.stream() - .map(BucketShards::getShards) - .flatMap(Collection::stream) - .map(ShardNodes::getShardUuid) - .collect(toSet()); - assertThat(actualAllUuids).isEqualTo(expectedAllUuids); - - // verify that conflicting updates are handled - newShards = ImmutableList.of(shardInfo(UUID.randomUUID(), nodes.get(0))); - List finalNewShards = newShards; - long newTransactionId = shardManager.beginTransaction(); - assertTrinoExceptionThrownBy(() -> shardManager.replaceShardUuids(newTransactionId, tableId, columns, replacedUuids, finalNewShards, OptionalLong.of(0))) - .hasErrorCode(TRANSACTION_CONFLICT) - .hasMessage("Table was updated by a different transaction. Please retry the operation."); - } - - @Test - public void testExternalBatches() - { - long tableId = createTable("test"); - Optional externalBatchId = Optional.of("foo"); - - List shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, externalBatchId, 0); - - shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); - - List finalShards = shards; - long newTransactionId = shardManager.beginTransaction(); - assertTrinoExceptionThrownBy(() -> shardManager.commitShards(newTransactionId, tableId, columns, finalShards, externalBatchId, 0)) - .hasErrorCode(RAPTOR_EXTERNAL_BATCH_ALREADY_EXISTS) - .hasMessage("External batch already exists: foo"); - } - - @Test - public void testBucketAssignments() - { - Node node1 = createTestingNode(); - Node node2 = createTestingNode(); - Node node3 = createTestingNode(); - - TestingTicker ticker = new TestingTicker(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - int bucketCount = 13; - long distributionId = metadataDao.insertDistribution(null, "test", bucketCount); - - Set originalNodes = ImmutableSet.of(node1, node2); - ShardManager shardManager = createShardManager(dbi, () -> originalNodes, ticker); - - shardManager.createBuckets(distributionId, bucketCount); - - List assignments = shardManager.getBucketAssignments(distributionId); - assertThat(assignments.size()).isEqualTo(bucketCount); - assertThat(ImmutableSet.copyOf(assignments)).isEqualTo(nodeIds(originalNodes)); - - Set newNodes = ImmutableSet.of(node1, node3); - shardManager = createShardManager(dbi, () -> newNodes, ticker); - - ShardManager finalShardManager = shardManager; - assertTrinoExceptionThrownBy(() -> finalShardManager.getBucketAssignments(distributionId)) - .hasErrorCode(SERVER_STARTING_UP) - .hasMessage("Cannot reassign buckets while server is starting"); - - ticker.increment(2, DAYS); - assignments = shardManager.getBucketAssignments(distributionId); - assertThat(assignments.size()).isEqualTo(bucketCount); - assertThat(ImmutableSet.copyOf(assignments)).isEqualTo(nodeIds(newNodes)); - - Set singleNode = ImmutableSet.of(node1); - shardManager = createShardManager(dbi, () -> singleNode, ticker); - ticker.increment(2, DAYS); - assignments = shardManager.getBucketAssignments(distributionId); - assertThat(assignments.size()).isEqualTo(bucketCount); - assertThat(ImmutableSet.copyOf(assignments)).isEqualTo(nodeIds(singleNode)); - } - - @Test - public void testEmptyTable() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - try (ResultIterator iterator = shardManager.getShardNodes(tableId, TupleDomain.all())) { - assertThat(iterator.hasNext()).isFalse(); - } - } - - @Test - public void testEmptyTableBucketed() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, true, OptionalLong.empty()); - - try (ResultIterator iterator = shardManager.getShardNodesBucketed(tableId, true, ImmutableList.of(), TupleDomain.all())) { - assertThat(iterator.hasNext()).isFalse(); - } - } - - @Test - public void testTemporalColumnTableCreation() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, TIMESTAMP_MILLIS)); - shardManager.createTable(tableId, columns, false, OptionalLong.of(1)); - - long tableId2 = createTable("test2"); - List columns2 = ImmutableList.of(new ColumnInfo(1, TIMESTAMP_MILLIS)); - shardManager.createTable(tableId2, columns2, true, OptionalLong.of(1)); - } - - @Test - public void testShardPruning() - { - ShardInfo shard1 = shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.builder() - .add(new ColumnStats(1, 5, 10)) - .add(new ColumnStats(2, -20.0, 20.0)) - .add(new ColumnStats(3, date(2013, 5, 11), date(2013, 6, 13))) - .add(new ColumnStats(4, timestamp(2013, 5, 11, 4, 5, 6), timestamp(2013, 6, 13, 7, 8, 9))) - .add(new ColumnStats(5, "hello", "world")) - .add(new ColumnStats(6, false, true)) - .build()); - - ShardInfo shard2 = shardInfo( - UUID.randomUUID(), - "node2", - ImmutableList.builder() - .add(new ColumnStats(1, 2, 8)) - .add(new ColumnStats(2, null, 50.0)) - .add(new ColumnStats(3, date(2012, 1, 1), date(2012, 12, 31))) - .add(new ColumnStats(4, timestamp(2012, 1, 1, 2, 3, 4), timestamp(2012, 12, 31, 5, 6, 7))) - .add(new ColumnStats(5, "cat", "dog")) - .add(new ColumnStats(6, true, true)) - .build()); - - ShardInfo shard3 = shardInfo( - UUID.randomUUID(), - "node3", - ImmutableList.builder() - .add(new ColumnStats(1, 15, 20)) - .add(new ColumnStats(2, null, null)) - .add(new ColumnStats(3, date(2013, 4, 1), date(2013, 6, 1))) - .add(new ColumnStats(4, timestamp(2013, 4, 1, 8, 7, 6), timestamp(2013, 6, 1, 6, 5, 4))) - .add(new ColumnStats(5, "grape", "orange")) - .add(new ColumnStats(6, false, false)) - .build()); - - List shards = ImmutableList.builder() - .add(shard1) - .add(shard2) - .add(shard3) - .build(); - - List columns = ImmutableList.builder() - .add(new ColumnInfo(1, BIGINT)) - .add(new ColumnInfo(2, DOUBLE)) - .add(new ColumnInfo(3, DATE)) - .add(new ColumnInfo(4, TIMESTAMP_MILLIS)) - .add(new ColumnInfo(5, createVarcharType(10))) - .add(new ColumnInfo(6, BOOLEAN)) - .add(new ColumnInfo(7, VARBINARY)) - .build(); - - RaptorColumnHandle c1 = new RaptorColumnHandle("c1", 1, BIGINT); - RaptorColumnHandle c2 = new RaptorColumnHandle("c2", 2, DOUBLE); - RaptorColumnHandle c3 = new RaptorColumnHandle("c3", 3, DATE); - RaptorColumnHandle c4 = new RaptorColumnHandle("c4", 4, TIMESTAMP_MILLIS); - RaptorColumnHandle c5 = new RaptorColumnHandle("c5", 5, createVarcharType(10)); - RaptorColumnHandle c6 = new RaptorColumnHandle("c6", 6, BOOLEAN); - - long tableId = createTable("test"); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - shardAssertion(tableId).expected(shards); - - shardAssertion(tableId).equal(c1, BIGINT, 3L).expected(shard2); - shardAssertion(tableId).equal(c1, BIGINT, 8L).expected(shard1, shard2); - shardAssertion(tableId).equal(c1, BIGINT, 9L).expected(shard1); - shardAssertion(tableId).equal(c1, BIGINT, 13L).expected(); - shardAssertion(tableId).between(c1, BIGINT, 8L, 14L).expected(shard1, shard2); - shardAssertion(tableId).between(c1, BIGINT, 8L, 15L).expected(shards); - shardAssertion(tableId).between(c1, BIGINT, 8L, 16L).expected(shards); - shardAssertion(tableId).between(c1, BIGINT, 12L, 14L).expected(); - shardAssertion(tableId).between(c1, BIGINT, 5L, 10L).expected(shard1, shard2); - shardAssertion(tableId).between(c1, BIGINT, 16L, 18L).expected(shard3); - shardAssertion(tableId).between(c1, BIGINT, 1L, 25L).expected(shards); - shardAssertion(tableId).between(c1, BIGINT, 4L, 12L).expected(shard1, shard2); - shardAssertion(tableId).range(c1, lessThan(BIGINT, 5L)).expected(shard1, shard2); - shardAssertion(tableId).range(c1, lessThan(BIGINT, 4L)).expected(shard2); - shardAssertion(tableId).range(c1, lessThan(BIGINT, 11L)).expected(shard1, shard2); - shardAssertion(tableId).range(c1, lessThan(BIGINT, 25L)).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(BIGINT, 1L)).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(BIGINT, 8L)).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(BIGINT, 9L)).expected(shard1, shard3); - - shardAssertion(tableId) - .between(c1, BIGINT, -25L, 25L) - .between(c2, DOUBLE, -1000.0, 1000.0) - .between(c3, BIGINT, 0L, 50000L) - .between(c4, TIMESTAMP_MILLIS, 0L, timestamp(2015, 1, 2, 3, 4, 5)) - .between(c5, createVarcharType(10), utf8Slice("a"), utf8Slice("zzzzz")) - .between(c6, BOOLEAN, false, true) - .expected(shards); - - shardAssertion(tableId) - .between(c1, BIGINT, 4L, 12L) - .between(c3, DATE, date(2013, 3, 3), date(2013, 5, 25)) - .expected(shard1); - - shardAssertion(tableId).equal(c2, DOUBLE, 25.0).expected(shard2, shard3); - shardAssertion(tableId).equal(c2, DOUBLE, 50.1).expected(shard3); - - shardAssertion(tableId).equal(c3, DATE, date(2013, 5, 12)).expected(shard1, shard3); - - shardAssertion(tableId).range(c4, greaterThan(TIMESTAMP_MILLIS, timestamp(2013, 1, 1, 0, 0, 0))).expected(shard1, shard3); - - shardAssertion(tableId).between(c5, createVarcharType(10), utf8Slice("cow"), utf8Slice("milk")).expected(shards); - shardAssertion(tableId).equal(c5, createVarcharType(10), utf8Slice("fruit")).expected(); - shardAssertion(tableId).equal(c5, createVarcharType(10), utf8Slice("pear")).expected(shard1); - shardAssertion(tableId).equal(c5, createVarcharType(10), utf8Slice("cat")).expected(shard2); - shardAssertion(tableId).range(c5, greaterThan(createVarcharType(10), utf8Slice("gum"))).expected(shard1, shard3); - shardAssertion(tableId).range(c5, lessThan(createVarcharType(10), utf8Slice("air"))).expected(); - - shardAssertion(tableId).equal(c6, BOOLEAN, true).expected(shard1, shard2); - shardAssertion(tableId).equal(c6, BOOLEAN, false).expected(shard1, shard3); - shardAssertion(tableId).range(c6, greaterThanOrEqual(BOOLEAN, false)).expected(shards); - shardAssertion(tableId).range(c6, lessThan(BOOLEAN, true)).expected(shards); - shardAssertion(tableId).range(c6, lessThan(BOOLEAN, false)).expected(shard1, shard3); - - // Test multiple ranges - shardAssertion(tableId) - .domain(c1, createDomain(lessThan(BIGINT, 0L), greaterThan(BIGINT, 25L))) - .expected(); - - shardAssertion(tableId) - .domain(c1, createDomain(range(BIGINT, 3L, true, 4L, true), range(BIGINT, 16L, true, 18L, true))) - .expected(shard2, shard3); - - shardAssertion(tableId) - .domain(c5, createDomain( - range(createVarcharType(10), utf8Slice("gum"), true, utf8Slice("happy"), true), - range(createVarcharType(10), utf8Slice("pear"), true, utf8Slice("wall"), true))) - .expected(shard1, shard3); - - shardAssertion(tableId) - .domain(c1, createDomain(range(BIGINT, 3L, true, 4L, true), range(BIGINT, 16L, true, 18L, true))) - .domain(c5, createDomain( - range(createVarcharType(10), utf8Slice("gum"), true, utf8Slice("happy"), true), - range(createVarcharType(10), utf8Slice("pear"), true, utf8Slice("wall"), true))) - .expected(shard3); - } - - @Test - public void testShardPruningTruncatedValues() - { - String prefix = "x".repeat(MAX_BINARY_INDEX_SIZE); - - ColumnStats stats = new ColumnStats(1, prefix + "a", prefix + "z"); - ShardInfo shard = shardInfo(UUID.randomUUID(), "node", ImmutableList.of(stats)); - - List shards = ImmutableList.of(shard); - - List columns = ImmutableList.of(new ColumnInfo(1, createVarcharType(10))); - RaptorColumnHandle c1 = new RaptorColumnHandle("c1", 1, createVarcharType(10)); - - long tableId = createTable("test"); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - shardAssertion(tableId).expected(shards); - shardAssertion(tableId).equal(c1, createVarcharType(10), utf8Slice(prefix)).expected(shards); - shardAssertion(tableId).equal(c1, createVarcharType(10), utf8Slice(prefix + "c")).expected(shards); - shardAssertion(tableId).range(c1, lessThan(createVarcharType(10), utf8Slice(prefix + "c"))).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(createVarcharType(10), utf8Slice(prefix + "zzz"))).expected(shards); - - shardAssertion(tableId).between(c1, createVarcharType(10), utf8Slice("w"), utf8Slice("y")).expected(shards); - shardAssertion(tableId).range(c1, greaterThan(createVarcharType(10), utf8Slice("x"))).expected(shards); - - shardAssertion(tableId).between(c1, createVarcharType(10), utf8Slice("x"), utf8Slice("x")).expected(); - shardAssertion(tableId).range(c1, lessThan(createVarcharType(10), utf8Slice("w"))).expected(); - shardAssertion(tableId).range(c1, lessThan(createVarcharType(10), utf8Slice("x"))).expected(); - shardAssertion(tableId).range(c1, greaterThan(createVarcharType(10), utf8Slice("y"))).expected(); - - Slice shorter = utf8Slice(prefix.substring(0, prefix.length() - 1)); - shardAssertion(tableId).equal(c1, createVarcharType(10), shorter).expected(); - shardAssertion(tableId).range(c1, lessThan(createVarcharType(10), shorter)).expected(); - shardAssertion(tableId).range(c1, greaterThan(createVarcharType(10), shorter)).expected(shards); - } - - @Test - public void testShardPruningNoStats() - { - ShardInfo shard = shardInfo(UUID.randomUUID(), "node"); - List shards = ImmutableList.of(shard); - - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - RaptorColumnHandle c1 = new RaptorColumnHandle("c1", 1, BIGINT); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - shardAssertion(tableId).expected(shards); - shardAssertion(tableId).equal(c1, BIGINT, 3L).expected(shards); - } - - @Test - public void testAddNewColumn() - throws Exception - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - int before = columnCount(tableId); - - ColumnInfo newColumn = new ColumnInfo(2, BIGINT); - shardManager.addColumn(tableId, newColumn); - int after = columnCount(tableId); - // should be 2 more: min and max columns - assertThat(after).isEqualTo(before + 2); - } - - @Test - public void testAddDuplicateColumn() - throws Exception - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - int before = columnCount(tableId); - - shardManager.addColumn(tableId, columns.get(0)); - int after = columnCount(tableId); - // no error, no columns added - assertThat(after).isEqualTo(before); - } - - @Test - public void testMaintenanceBlocked() - { - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - Set oldShards = ImmutableSet.of(UUID.randomUUID()); - - dbi.onDemand(MetadataDao.class).blockMaintenance(tableId); - - long transactionId = shardManager.beginTransaction(); - assertTrinoExceptionThrownBy(() -> shardManager.replaceShardUuids(transactionId, tableId, columns, oldShards, ImmutableSet.of(), OptionalLong.empty())) - .hasErrorCode(TRANSACTION_CONFLICT) - .hasMessage("Maintenance is blocked for table"); - } - - private Set getShardNodes(long tableId, TupleDomain predicate) - { - try (ResultIterator iterator = shardManager.getShardNodes(tableId, predicate)) { - return ImmutableSet.copyOf(concat(transform(iterator, i -> i.getShards().iterator()))); - } - } - - private long createTable(String name) - { - return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0); - } - - public static ShardInfo shardInfo(UUID shardUuid, String nodeIdentifier) - { - return shardInfo(shardUuid, nodeIdentifier, ImmutableList.of()); - } - - public static ShardInfo shardInfo(UUID shardUuid, String nodeId, List columnStats) - { - return new ShardInfo(shardUuid, OptionalInt.empty(), ImmutableSet.of(nodeId), columnStats, 0, 0, 0, 0); - } - - private static Set toShardNodes(List shards) - { - return shards.stream() - .map(shard -> new ShardNodes(shard.getShardUuid(), shard.getNodeIdentifiers())) - .collect(toSet()); - } - - public static ShardManager createShardManager(Jdbi dbi) - { - return createShardManager(dbi, ImmutableSet::of, systemTicker()); - } - - public static ShardManager createShardManager(Jdbi dbi, NodeSupplier nodeSupplier) - { - return createShardManager(dbi, nodeSupplier, systemTicker()); - } - - public static ShardManager createShardManager(Jdbi dbi, NodeSupplier nodeSupplier, Ticker ticker) - { - DaoSupplier shardDaoSupplier = new DaoSupplier<>(dbi, H2ShardDao.class); - AssignmentLimiter assignmentLimiter = new AssignmentLimiter(nodeSupplier, ticker, new MetadataConfig()); - return new DatabaseShardManager(dbi, shardDaoSupplier, nodeSupplier, assignmentLimiter, ticker, new Duration(1, DAYS)); - } - - private static Domain createDomain(Range first, Range... ranges) - { - return Domain.create(ValueSet.ofRanges(first, ranges), false); - } - - private ShardAssertion shardAssertion(long tableId) - { - return new ShardAssertion(tableId); - } - - private class ShardAssertion - { - private final Map domains = new HashMap<>(); - private final long tableId; - - public ShardAssertion(long tableId) - { - this.tableId = tableId; - } - - public ShardAssertion domain(RaptorColumnHandle column, Domain domain) - { - domains.put(column, domain); - return this; - } - - public ShardAssertion range(RaptorColumnHandle column, Range range) - { - return domain(column, createDomain(range)); - } - - public ShardAssertion equal(RaptorColumnHandle column, Type type, Object value) - { - return domain(column, Domain.singleValue(type, value)); - } - - public ShardAssertion between(RaptorColumnHandle column, Type type, Object low, Object high) - { - return range(column, Range.range(type, low, true, high, true)); - } - - public void expected(ShardInfo... shards) - { - expected(ImmutableList.copyOf(shards)); - } - - public void expected(List shards) - { - TupleDomain predicate = TupleDomain.withColumnDomains(domains); - Set actual = getShardNodes(tableId, predicate); - assertThat(actual).isEqualTo(toShardNodes(shards)); - } - } - - private static long date(int year, int month, int day) - { - return LocalDate.of(year, month, day).toEpochDay(); - } - - private static long timestamp(int year, int month, int day, int hour, int minute, int second) - { - return ZonedDateTime.of(year, month, day, hour, minute, second, 0, UTC).toInstant().toEpochMilli(); - } - - private static Set nodeIds(Collection nodes) - { - return nodes.stream().map(Node::getNodeIdentifier).collect(toSet()); - } - - private static Node createTestingNode() - { - return new InternalNode(UUID.randomUUID().toString(), URI.create("http://test"), NodeVersion.UNKNOWN, false); - } - - private int columnCount(long tableId) - throws SQLException - { - try (Statement statement = dummyHandle.getConnection().createStatement()) { - try (ResultSet rs = statement.executeQuery(format("SELECT * FROM %s LIMIT 0", shardIndexTable(tableId)))) { - return rs.getMetaData().getColumnCount(); - } - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestH2DatabaseConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestH2DatabaseConfig.java deleted file mode 100644 index 8cdd7155c491..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestH2DatabaseConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestH2DatabaseConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(H2DatabaseConfig.class) - .setFilename(null)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("metadata.db.filename", "/tmp/db"); - - H2DatabaseConfig expected = new H2DatabaseConfig() - .setFilename("/tmp/db"); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataConfig.java deleted file mode 100644 index e8e3ecafc2da..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.MINUTES; - -public class TestMetadataConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(MetadataConfig.class) - .setStartupGracePeriod(new Duration(5, MINUTES)) - .setReassignmentDelay(new Duration(0, MINUTES)) - .setReassignmentInterval(new Duration(0, MINUTES))); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("raptor.startup-grace-period", "42m") - .put("raptor.reassignment-delay", "6m") - .put("raptor.reassignment-interval", "7m") - .buildOrThrow(); - - MetadataConfig expected = new MetadataConfig() - .setStartupGracePeriod(new Duration(42, MINUTES)) - .setReassignmentDelay(new Duration(6, MINUTES)) - .setReassignmentInterval(new Duration(7, MINUTES)); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataDao.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataDao.java deleted file mode 100644 index 8e6ed4e46048..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestMetadataDao.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestMetadataDao -{ - private MetadataDao dao; - private Handle dummyHandle; - - @BeforeEach - public void setup() - { - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - dao = dbi.onDemand(MetadataDao.class); - createTablesWithRetry(dbi); - } - - @AfterEach - public void tearDown() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testTemporalColumn() - { - Long columnId = 1L; - long tableId = dao.insertTable("schema1", "table1", true, false, null, 0); - dao.insertColumn(tableId, columnId, "col1", 1, "bigint", null, null); - Long temporalColumnId = dao.getTemporalColumnId(tableId); - assertThat(temporalColumnId).isNull(); - - dao.updateTemporalColumnId(tableId, columnId); - temporalColumnId = dao.getTemporalColumnId(tableId); - assertThat(temporalColumnId).isNotNull(); - assertThat(temporalColumnId).isEqualTo(columnId); - - long tableId2 = dao.insertTable("schema1", "table2", true, false, null, 0); - Long columnId2 = dao.getTemporalColumnId(tableId2); - assertThat(columnId2).isNull(); - } - - @Test - public void testGetTableInformation() - { - Long columnId = 1L; - long tableId = dao.insertTable("schema1", "table1", true, false, null, 0); - dao.insertColumn(tableId, columnId, "col1", 1, "bigint", null, null); - - Table info = dao.getTableInformation(tableId); - assertTable(info, tableId); - - info = dao.getTableInformation("schema1", "table1"); - assertTable(info, tableId); - } - - private static void assertTable(Table info, long tableId) - { - assertThat(info.getTableId()).isEqualTo(tableId); - assertThat(info.getDistributionId()).isEqualTo(Optional.empty()); - assertThat(info.getDistributionName()).isEqualTo(Optional.empty()); - assertThat(info.getBucketCount()).isEqualTo(OptionalInt.empty()); - assertThat(info.getTemporalColumnId()).isEqualTo(OptionalLong.empty()); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorMetadata.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorMetadata.java deleted file mode 100644 index 74dd0f6c3b1c..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorMetadata.java +++ /dev/null @@ -1,826 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.trino.metadata.MetadataUtil.TableMetadataBuilder; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.RaptorMetadata; -import io.trino.plugin.raptor.legacy.RaptorPartitioningHandle; -import io.trino.plugin.raptor.legacy.RaptorSessionProperties; -import io.trino.plugin.raptor.legacy.RaptorTableHandle; -import io.trino.plugin.raptor.legacy.storage.StorageManagerConfig; -import io.trino.spi.NodeManager; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableLayout; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorViewDefinition; -import io.trino.spi.connector.ConnectorViewDefinition.ViewColumn; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.testing.TestingConnectorSession; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.stream.Collectors; - -import static com.google.common.base.Ticker.systemTicker; -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.BUCKETED_ON_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.BUCKET_COUNT_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.DISTRIBUTION_NAME_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.ORDERING_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.ORGANIZED_PROPERTY; -import static io.trino.plugin.raptor.legacy.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.spi.StandardErrorCode.TRANSACTION_CONFLICT; -import static io.trino.spi.connector.RetryMode.NO_RETRIES; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder; -import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestRaptorMetadata -{ - private static final SchemaTableName DEFAULT_TEST_ORDERS = new SchemaTableName("test", "orders"); - private static final SchemaTableName DEFAULT_TEST_LINEITEMS = new SchemaTableName("test", "lineitems"); - private static final ConnectorSession SESSION = TestingConnectorSession.builder() - .setPropertyMetadata(new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties()) - .build(); - - private Jdbi dbi; - private Handle dummyHandle; - private ShardManager shardManager; - private RaptorMetadata metadata; - - @BeforeEach - public void setupDatabase() - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - - NodeManager nodeManager = new TestingNodeManager(); - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - shardManager = createShardManager(dbi, nodeSupplier, systemTicker()); - metadata = new RaptorMetadata(dbi, shardManager); - } - - @AfterEach - public void cleanupDatabase() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testRenameColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - ColumnHandle columnHandle = metadata.getColumnHandles(SESSION, tableHandle).get("orderkey"); - - metadata.renameColumn(SESSION, raptorTableHandle, columnHandle, "orderkey_renamed"); - - assertThat(metadata.getColumnHandles(SESSION, tableHandle).get("orderkey")).isNull(); - assertThat(metadata.getColumnHandles(SESSION, tableHandle).get("orderkey_renamed")).isNotNull(); - } - - @Test - public void testAddColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, buildTable(ImmutableMap.of(), tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("price", BIGINT)), - SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - - metadata.addColumn(SESSION, raptorTableHandle, new ColumnMetadata("new_col", BIGINT)); - assertThat(metadata.getColumnHandles(SESSION, raptorTableHandle).get("new_col")).isNotNull(); - } - - @Test - public void testDropColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, buildTable(ImmutableMap.of(), tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("price", BIGINT)), - SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - - ColumnHandle lastColumn = metadata.getColumnHandles(SESSION, tableHandle).get("orderkey"); - metadata.dropColumn(SESSION, raptorTableHandle, lastColumn); - assertThat(metadata.getColumnHandles(SESSION, tableHandle).get("orderkey")).isNull(); - } - - @Test - public void testAddColumnAfterDropColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, buildTable(ImmutableMap.of(), tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("price", BIGINT)), - SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - ColumnHandle column = metadata.getColumnHandles(SESSION, tableHandle).get("orderkey"); - - metadata.dropColumn(SESSION, raptorTableHandle, column); - metadata.addColumn(SESSION, raptorTableHandle, new ColumnMetadata("new_col", BIGINT)); - assertThat(metadata.getColumnHandles(SESSION, tableHandle).get("orderkey")).isNull(); - assertThat(metadata.getColumnHandles(SESSION, raptorTableHandle).get("new_col")).isNotNull(); - } - - @Test - public void testDropColumnDisallowed() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - Map properties = ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 16, - BUCKETED_ON_PROPERTY, ImmutableList.of("orderkey"), - ORDERING_PROPERTY, ImmutableList.of("totalprice"), - TEMPORAL_COLUMN_PROPERTY, "orderdate"); - ConnectorTableMetadata ordersTable = buildTable(properties, tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("totalprice", DOUBLE) - .column("orderdate", DATE) - .column("highestid", BIGINT)); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle ordersTableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(ordersTableHandle, RaptorTableHandle.class); - RaptorTableHandle ordersRaptorTableHandle = (RaptorTableHandle) ordersTableHandle; - assertThat(ordersRaptorTableHandle.getTableId()).isEqualTo(1); - - assertInstanceOf(ordersRaptorTableHandle, RaptorTableHandle.class); - - // disallow dropping bucket, sort, temporal and highest-id columns - ColumnHandle bucketColumn = metadata.getColumnHandles(SESSION, ordersRaptorTableHandle).get("orderkey"); - assertThatThrownBy(() -> metadata.dropColumn(SESSION, ordersTableHandle, bucketColumn)) - .isInstanceOf(TrinoException.class) - .hasMessage("Cannot drop bucket columns"); - - ColumnHandle sortColumn = metadata.getColumnHandles(SESSION, ordersRaptorTableHandle).get("totalprice"); - assertThatThrownBy(() -> metadata.dropColumn(SESSION, ordersTableHandle, sortColumn)) - .isInstanceOf(TrinoException.class) - .hasMessage("Cannot drop sort columns"); - - ColumnHandle temporalColumn = metadata.getColumnHandles(SESSION, ordersRaptorTableHandle).get("orderdate"); - assertThatThrownBy(() -> metadata.dropColumn(SESSION, ordersTableHandle, temporalColumn)) - .isInstanceOf(TrinoException.class) - .hasMessage("Cannot drop the temporal column"); - - ColumnHandle highestColumn = metadata.getColumnHandles(SESSION, ordersRaptorTableHandle).get("highestid"); - assertThatThrownBy(() -> metadata.dropColumn(SESSION, ordersTableHandle, highestColumn)) - .isInstanceOf(TrinoException.class) - .hasMessage("Cannot drop the column which has the largest column ID in the table"); - } - - @Test - public void testRenameTable() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - SchemaTableName renamedTable = new SchemaTableName(raptorTableHandle.getSchemaName(), "orders_renamed"); - - metadata.renameTable(SESSION, raptorTableHandle, renamedTable); - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - ConnectorTableHandle renamedTableHandle = metadata.getTableHandle(SESSION, renamedTable, Optional.empty(), Optional.empty()); - assertThat(renamedTableHandle).isNotNull(); - assertThat(((RaptorTableHandle) renamedTableHandle).getTableName()).isEqualTo(renamedTable.getTableName()); - } - - @Test - public void testCreateTable() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - assertThat(((RaptorTableHandle) tableHandle).getTableId()).isEqualTo(1); - - ConnectorTableMetadata table = metadata.getTableMetadata(SESSION, tableHandle); - assertTableEqual(table, getOrdersTable()); - - ColumnHandle columnHandle = metadata.getColumnHandles(SESSION, tableHandle).get("orderkey"); - assertInstanceOf(columnHandle, RaptorColumnHandle.class); - assertThat(((RaptorColumnHandle) columnHandle).getColumnId()).isEqualTo(1); - - ColumnMetadata columnMetadata = metadata.getColumnMetadata(SESSION, tableHandle, columnHandle); - assertThat(columnMetadata).isNotNull(); - assertThat(columnMetadata.getName()).isEqualTo("orderkey"); - assertThat(columnMetadata.getType()).isEqualTo(BIGINT); - } - - @Test - public void testTableProperties() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of( - ORDERING_PROPERTY, ImmutableList.of("orderdate", "custkey"), - TEMPORAL_COLUMN_PROPERTY, "orderdate")); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // verify sort columns - List sortColumns = metadataDao.listSortColumns(tableId); - assertTableColumnsEqual(sortColumns, ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderdate", DATE, 4, 3, OptionalInt.empty(), OptionalInt.of(0), true), - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.empty(), OptionalInt.of(1), false))); - - // verify temporal column - assertThat(metadataDao.getTemporalColumnId(tableId)).isEqualTo(Long.valueOf(4)); - - // verify no organization - assertThat(metadataDao.getTableInformation(tableId).isOrganized()).isFalse(); - - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testTablePropertiesWithOrganization() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of( - ORDERING_PROPERTY, ImmutableList.of("orderdate", "custkey"), - ORGANIZED_PROPERTY, true)); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // verify sort columns - List sortColumns = metadataDao.listSortColumns(tableId); - assertTableColumnsEqual(sortColumns, ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderdate", DATE, 4, 3, OptionalInt.empty(), OptionalInt.of(0), false), - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.empty(), OptionalInt.of(1), false))); - - // verify organization - assertThat(metadataDao.getTableInformation(tableId).isOrganized()).isTrue(); - - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testCreateBucketedTable() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 16, - BUCKETED_ON_PROPERTY, ImmutableList.of("custkey", "orderkey"))); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - assertTableColumnsEqual(metadataDao.listBucketColumns(tableId), ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.of(0), OptionalInt.empty(), false), - new TableColumn(DEFAULT_TEST_ORDERS, "orderkey", BIGINT, 1, 0, OptionalInt.of(1), OptionalInt.empty(), false))); - - assertThat(raptorTableHandle.getBucketCount()).isEqualTo(OptionalInt.of(16)); - - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(1)); - - metadata.dropTable(SESSION, tableHandle); - - // create a new table and verify it has a different distribution - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - tableId = ((RaptorTableHandle) metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).getTableId(); - assertThat(tableId).isEqualTo(2); - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(2)); - } - - @Test - public void testCreateBucketedTableAsSelect() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 32, - BUCKETED_ON_PROPERTY, ImmutableList.of("orderkey", "custkey"))); - - ConnectorTableLayout layout = metadata.getNewTableLayout(SESSION, ordersTable).get(); - assertThat(layout.getPartitionColumns()).isEqualTo(ImmutableList.of("orderkey", "custkey")); - assertThat(layout.getPartitioning()).isPresent(); - assertInstanceOf(layout.getPartitioning().get(), RaptorPartitioningHandle.class); - RaptorPartitioningHandle partitioning = (RaptorPartitioningHandle) layout.getPartitioning().get(); - assertThat(partitioning.getDistributionId()).isEqualTo(1); - - ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, ordersTable, Optional.of(layout), NO_RETRIES, false); - metadata.finishCreateTable(SESSION, outputHandle, ImmutableList.of(), ImmutableList.of()); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - assertTableColumnsEqual(metadataDao.listBucketColumns(tableId), ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderkey", BIGINT, 1, 0, OptionalInt.of(0), OptionalInt.empty(), false), - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.of(1), OptionalInt.empty(), false))); - - assertThat(raptorTableHandle.getBucketCount()).isEqualTo(OptionalInt.of(32)); - - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(1)); - - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testCreateBucketedTableExistingDistribution() - { - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // create orders table - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata table = getOrdersTable(ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 16, - BUCKETED_ON_PROPERTY, ImmutableList.of("orderkey"), - DISTRIBUTION_NAME_PROPERTY, "orders")); - metadata.createTable(SESSION, table, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - - long tableId = raptorTableHandle.getTableId(); - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - assertTableColumnsEqual(metadataDao.listBucketColumns(tableId), ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderkey", BIGINT, 1, 0, OptionalInt.of(0), OptionalInt.empty(), false))); - - assertThat(raptorTableHandle.getBucketCount()).isEqualTo(OptionalInt.of(16)); - - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(1)); - - // create lineitems table - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_LINEITEMS, Optional.empty(), Optional.empty())).isNull(); - - table = getLineItemsTable(ImmutableMap.of( - BUCKET_COUNT_PROPERTY, 16, - BUCKETED_ON_PROPERTY, ImmutableList.of("orderkey"), - DISTRIBUTION_NAME_PROPERTY, "orders")); - metadata.createTable(SESSION, table, SaveMode.FAIL); - - tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_LINEITEMS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - raptorTableHandle = (RaptorTableHandle) tableHandle; - - tableId = raptorTableHandle.getTableId(); - assertThat(tableId).isEqualTo(2); - - assertTableColumnsEqual(metadataDao.listBucketColumns(tableId), ImmutableList.of( - new TableColumn(DEFAULT_TEST_LINEITEMS, "orderkey", BIGINT, 1, 0, OptionalInt.of(0), OptionalInt.empty(), false))); - - assertThat(raptorTableHandle.getBucketCount()).isEqualTo(OptionalInt.of(16)); - - assertThat(getTableDistributionId(tableId)).isEqualTo(Long.valueOf(1)); - } - - @Test - public void testInvalidOrderingColumns() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of(ORDERING_PROPERTY, ImmutableList.of("orderdatefoo"))), SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Ordering column does not exist: orderdatefoo"); - } - - @Test - public void testInvalidTemporalColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "foo")), SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Temporal column does not exist: foo"); - } - - @Test - public void testInvalidTemporalColumnType() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "orderkey")), SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Temporal column must be of type timestamp or date: orderkey"); - } - - @Test - public void testInvalidTemporalOrganization() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of( - TEMPORAL_COLUMN_PROPERTY, "orderdate", - ORGANIZED_PROPERTY, true)), - SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Table with temporal columns cannot be organized"); - } - - @Test - public void testInvalidOrderingOrganization() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - assertThatThrownBy(() -> metadata.createTable(SESSION, getOrdersTable(ImmutableMap.of(ORGANIZED_PROPERTY, true)), SaveMode.FAIL)) - .isInstanceOf(TrinoException.class) - .hasMessage("Table organization requires an ordering"); - } - - @Test - public void testSortOrderProperty() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of(ORDERING_PROPERTY, ImmutableList.of("orderdate", "custkey"))); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // verify sort columns - List sortColumns = metadataDao.listSortColumns(tableId); - assertTableColumnsEqual(sortColumns, ImmutableList.of( - new TableColumn(DEFAULT_TEST_ORDERS, "orderdate", DATE, 4, 3, OptionalInt.empty(), OptionalInt.of(0), false), - new TableColumn(DEFAULT_TEST_ORDERS, "custkey", BIGINT, 2, 1, OptionalInt.empty(), OptionalInt.of(1), false))); - - // verify temporal column is not set - assertThat(metadataDao.getTemporalColumnId(tableId)).isEqualTo(null); - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testTemporalColumn() - { - assertThat(metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty())).isNull(); - - ConnectorTableMetadata ordersTable = getOrdersTable(ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "orderdate")); - metadata.createTable(SESSION, ordersTable, SaveMode.FAIL); - - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - assertInstanceOf(tableHandle, RaptorTableHandle.class); - RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle; - assertThat(raptorTableHandle.getTableId()).isEqualTo(1); - - long tableId = raptorTableHandle.getTableId(); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - // verify sort columns are not set - List sortColumns = metadataDao.listSortColumns(tableId); - assertThat(sortColumns.size()).isEqualTo(0); - assertThat(sortColumns).isEqualTo(ImmutableList.of()); - - // verify temporal column is set - assertThat(metadataDao.getTemporalColumnId(tableId)).isEqualTo(Long.valueOf(4)); - metadata.dropTable(SESSION, tableHandle); - } - - @Test - public void testListTables() - { - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - List tables = metadata.listTables(SESSION, Optional.empty()); - assertThat(tables).isEqualTo(ImmutableList.of(DEFAULT_TEST_ORDERS)); - } - - @Test - public void testListTableColumns() - { - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - Map> columns = metadata.listTableColumns(SESSION, new SchemaTablePrefix()); - assertThat(columns).isEqualTo(ImmutableMap.of(DEFAULT_TEST_ORDERS, getOrdersTable().getColumns())); - } - - @Test - public void testListTableColumnsFiltering() - { - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - Map> filterCatalog = metadata.listTableColumns(SESSION, new SchemaTablePrefix()); - Map> filterSchema = metadata.listTableColumns(SESSION, new SchemaTablePrefix("test")); - Map> filterTable = metadata.listTableColumns(SESSION, new SchemaTablePrefix("test", "orders")); - assertThat(filterCatalog).isEqualTo(filterSchema); - assertThat(filterCatalog).isEqualTo(filterTable); - } - - @Test - public void testViews() - { - SchemaTableName test1 = new SchemaTableName("test", "test_view1"); - SchemaTableName test2 = new SchemaTableName("test", "test_view2"); - - // create views - metadata.createView(SESSION, test1, testingViewDefinition("test1"), ImmutableMap.of(), false); - metadata.createView(SESSION, test2, testingViewDefinition("test2"), ImmutableMap.of(), false); - - // verify listing - List list = metadata.listViews(SESSION, Optional.of("test")); - assertEqualsIgnoreOrder(list, ImmutableList.of(test1, test2)); - - // verify getting data - Map views = metadata.getViews(SESSION, Optional.of("test")); - assertThat(views.keySet()).isEqualTo(ImmutableSet.of(test1, test2)); - assertThat(views.get(test1).getOriginalSql()).isEqualTo("test1"); - assertThat(views.get(test2).getOriginalSql()).isEqualTo("test2"); - - // drop first view - metadata.dropView(SESSION, test1); - - assertThat(metadata.getViews(SESSION, Optional.of("test"))) - .containsOnlyKeys(test2); - - // drop second view - metadata.dropView(SESSION, test2); - - assertThat(metadata.getViews(SESSION, Optional.of("test"))) - .isEmpty(); - - // verify listing everything - assertThat(metadata.getViews(SESSION, Optional.empty())) - .isEmpty(); - } - - @Test - public void testCreateViewWithoutReplace() - { - SchemaTableName test = new SchemaTableName("test", "test_view"); - metadata.createView(SESSION, test, testingViewDefinition("test"), ImmutableMap.of(), false); - assertThatThrownBy(() -> metadata.createView(SESSION, test, testingViewDefinition("test"), ImmutableMap.of(), false)) - .isInstanceOf(TrinoException.class) - .hasMessage("View already exists: test.test_view"); - } - - @Test - public void testCreateViewWithReplace() - { - SchemaTableName test = new SchemaTableName("test", "test_view"); - - metadata.createView(SESSION, test, testingViewDefinition("aaa"), ImmutableMap.of(), true); - metadata.createView(SESSION, test, testingViewDefinition("bbb"), ImmutableMap.of(), true); - - assertThat(metadata.getView(SESSION, test)) - .map(ConnectorViewDefinition::getOriginalSql) - .contains("bbb"); - } - - @Test - public void testTransactionTableWrite() - { - // start table creation - long transactionId = 1; - ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, getOrdersTable(), Optional.empty(), NO_RETRIES, false); - - // transaction is in progress - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isNull(); - - // commit table creation - metadata.finishCreateTable(SESSION, outputHandle, ImmutableList.of(), ImmutableList.of()); - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isTrue(); - } - - @Test - public void testTransactionInsert() - { - // creating a table allocates a transaction - long transactionId = 1; - metadata.createTable(SESSION, getOrdersTable(), SaveMode.FAIL); - assertThat(transactionSuccessful(transactionId)).isTrue(); - - // start insert - transactionId++; - ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, DEFAULT_TEST_ORDERS, Optional.empty(), Optional.empty()); - ConnectorInsertTableHandle insertHandle = metadata.beginInsert(SESSION, tableHandle, ImmutableList.of(), NO_RETRIES); - - // transaction is in progress - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isNull(); - - // commit insert - metadata.finishInsert(SESSION, insertHandle, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()); - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isTrue(); - } - - @Test - public void testTransactionAbort() - { - // start table creation - long transactionId = 1; - ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, getOrdersTable(), Optional.empty(), NO_RETRIES, false); - - // transaction is in progress - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isNull(); - - // force transaction to abort - shardManager.rollbackTransaction(transactionId); - assertThat(transactionExists(transactionId)).isTrue(); - assertThat(transactionSuccessful(transactionId)).isFalse(); - - // commit table creation - assertTrinoExceptionThrownBy(() -> metadata.finishCreateTable(SESSION, outputHandle, ImmutableList.of(), ImmutableList.of())) - .hasErrorCode(TRANSACTION_CONFLICT) - .hasMessage("Transaction commit failed. Please retry the operation."); - } - - private boolean transactionExists(long transactionId) - { - return dbi.withHandle(handle -> handle - .select("SELECT count(*) FROM transactions WHERE transaction_id = ?", transactionId) - .mapTo(boolean.class) - .one()); - } - - private Boolean transactionSuccessful(long transactionId) - { - return dbi.withHandle(handle -> handle - .select("SELECT successful FROM transactions WHERE transaction_id = ?", transactionId) - .mapTo(Boolean.class) - .findFirst() - .orElse(null)); - } - - private Long getTableDistributionId(long tableId) - { - return dbi.withHandle(handle -> handle - .select("SELECT distribution_id FROM tables WHERE table_id = ?", tableId) - .mapTo(Long.class) - .findFirst() - .orElse(null)); - } - - private static ConnectorTableMetadata getOrdersTable() - { - return getOrdersTable(ImmutableMap.of()); - } - - private static ConnectorTableMetadata getOrdersTable(Map properties) - { - return buildTable(properties, tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("custkey", BIGINT) - .column("totalprice", DOUBLE) - .column("orderdate", DATE)); - } - - private static ConnectorTableMetadata getLineItemsTable(Map properties) - { - return buildTable(properties, tableMetadataBuilder(DEFAULT_TEST_LINEITEMS) - .column("orderkey", BIGINT) - .column("partkey", BIGINT) - .column("quantity", DOUBLE) - .column("price", DOUBLE)); - } - - private static ConnectorTableMetadata buildTable(Map properties, TableMetadataBuilder builder) - { - if (!properties.isEmpty()) { - for (Map.Entry entry : properties.entrySet()) { - builder.property(entry.getKey(), entry.getValue()); - } - } - return builder.build(); - } - - private static ConnectorViewDefinition testingViewDefinition(String sql) - { - return new ConnectorViewDefinition( - sql, - Optional.empty(), - Optional.empty(), - ImmutableList.of(new ViewColumn("test", BIGINT.getTypeId(), Optional.empty())), - Optional.empty(), - Optional.empty(), - true, - ImmutableList.of()); - } - - private static void assertTableEqual(ConnectorTableMetadata actual, ConnectorTableMetadata expected) - { - assertThat(actual.getTable()).isEqualTo(expected.getTable()); - - List actualColumns = actual.getColumns().stream() - .filter(columnMetadata -> !columnMetadata.isHidden()) - .collect(Collectors.toList()); - - List expectedColumns = expected.getColumns(); - assertThat(actualColumns.size()).isEqualTo(expectedColumns.size()); - for (int i = 0; i < actualColumns.size(); i++) { - ColumnMetadata actualColumn = actualColumns.get(i); - ColumnMetadata expectedColumn = expectedColumns.get(i); - assertThat(actualColumn.getName()).isEqualTo(expectedColumn.getName()); - assertThat(actualColumn.getType()).isEqualTo(expectedColumn.getType()); - } - assertThat(actual.getProperties()).isEqualTo(expected.getProperties()); - } - - private static void assertTableColumnEqual(TableColumn actual, TableColumn expected) - { - assertThat(actual.getTable()).isEqualTo(expected.getTable()); - assertThat(actual.getColumnId()).isEqualTo(expected.getColumnId()); - assertThat(actual.getColumnName()).isEqualTo(expected.getColumnName()); - assertThat(actual.getDataType()).isEqualTo(expected.getDataType()); - assertThat(actual.getOrdinalPosition()).isEqualTo(expected.getOrdinalPosition()); - assertThat(actual.getBucketOrdinal()).isEqualTo(expected.getBucketOrdinal()); - assertThat(actual.getSortOrdinal()).isEqualTo(expected.getSortOrdinal()); - assertThat(actual.isTemporal()).isEqualTo(expected.isTemporal()); - } - - private static void assertTableColumnsEqual(List actual, List expected) - { - assertThat(actual.size()).isEqualTo(expected.size()); - for (int i = 0; i < actual.size(); i++) { - assertTableColumnEqual(actual.get(i), expected.get(i)); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorSplitManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorSplitManager.java deleted file mode 100644 index 22f89a61c7bc..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestRaptorSplitManager.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.units.Duration; -import io.trino.client.NodeVersion; -import io.trino.metadata.InternalNode; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.RaptorMetadata; -import io.trino.plugin.raptor.legacy.RaptorSplitManager; -import io.trino.plugin.raptor.legacy.RaptorTableHandle; -import io.trino.plugin.raptor.legacy.RaptorTransactionHandle; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.spi.TrinoException; -import io.trino.spi.catalog.CatalogName; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorSplitSource; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.BigintType; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static com.google.common.base.Ticker.systemTicker; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.DatabaseShardManager.shardIndexTable; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.shardInfo; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestRaptorSplitManager -{ - private static final ConnectorTableMetadata TEST_TABLE = tableMetadataBuilder(new SchemaTableName("demo", "test_table")) - .column("ds", createVarcharType(10)) - .column("foo", createVarcharType(10)) - .column("bar", BigintType.BIGINT) - .build(); - - private Handle dummyHandle; - private Path temporary; - private RaptorMetadata metadata; - private RaptorSplitManager raptorSplitManager; - private ConnectorTableHandle tableHandle; - private ShardManager shardManager; - private long tableId; - - @BeforeEach - public void setup() - throws Exception - { - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - temporary = createTempDirectory(null); - AssignmentLimiter assignmentLimiter = new AssignmentLimiter(ImmutableSet::of, systemTicker(), new MetadataConfig()); - shardManager = new DatabaseShardManager(dbi, new DaoSupplier<>(dbi, ShardDao.class), ImmutableSet::of, assignmentLimiter, systemTicker(), new Duration(0, MINUTES)); - TestingNodeManager nodeManager = new TestingNodeManager(); - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - - String nodeName = UUID.randomUUID().toString(); - nodeManager.addNode(new InternalNode(nodeName, new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false)); - - CatalogName connectorId = new CatalogName("raptor"); - metadata = new RaptorMetadata(dbi, shardManager); - - metadata.createTable(SESSION, TEST_TABLE, SaveMode.FAIL); - tableHandle = metadata.getTableHandle(SESSION, TEST_TABLE.getTable(), Optional.empty(), Optional.empty()); - - List shards = ImmutableList.builder() - .add(shardInfo(UUID.randomUUID(), nodeName)) - .add(shardInfo(UUID.randomUUID(), nodeName)) - .add(shardInfo(UUID.randomUUID(), nodeName)) - .add(shardInfo(UUID.randomUUID(), nodeName)) - .build(); - - tableId = ((RaptorTableHandle) tableHandle).getTableId(); - - List columns = metadata.getColumnHandles(SESSION, tableHandle).values().stream() - .map(RaptorColumnHandle.class::cast) - .map(ColumnInfo::fromHandle) - .collect(toList()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - raptorSplitManager = new RaptorSplitManager(connectorId, nodeSupplier, shardManager, false); - } - - @AfterEach - public void teardown() - throws IOException - { - dummyHandle.close(); - dummyHandle = null; - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testSanity() - { - ConnectorSplitSource splitSource = getSplits(raptorSplitManager, tableHandle); - int splitCount = 0; - while (!splitSource.isFinished()) { - splitCount += getSplits(splitSource, 1000).size(); - } - assertThat(splitCount).isEqualTo(4); - } - - @Test - public void testNoHostForShard() - { - assertThatThrownBy(() -> { - deleteShardNodes(); - - ConnectorSplitSource splitSource = getSplits(raptorSplitManager, tableHandle); - getSplits(splitSource, 1000); - }) - .isInstanceOf(TrinoException.class) - .hasMessageMatching("No host for shard .* found: \\[\\]"); - } - - @Test - public void testAssignRandomNodeWhenBackupAvailable() - { - TestingNodeManager nodeManager = new TestingNodeManager(); - CatalogName connectorId = new CatalogName("raptor"); - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - InternalNode node = new InternalNode(UUID.randomUUID().toString(), URI.create("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); - nodeManager.addNode(node); - RaptorSplitManager raptorSplitManagerWithBackup = new RaptorSplitManager(connectorId, nodeSupplier, shardManager, true); - - deleteShardNodes(); - - ConnectorSplitSource partitionSplit = getSplits(raptorSplitManagerWithBackup, tableHandle); - List batch = getSplits(partitionSplit, 1); - assertThat(getOnlyElement(getOnlyElement(batch).getAddresses())).isEqualTo(node.getHostAndPort()); - } - - @Test - public void testNoNodes() - { - assertThatThrownBy(() -> { - deleteShardNodes(); - - RaptorSplitManager raptorSplitManagerWithBackup = new RaptorSplitManager(new CatalogName("fbraptor"), ImmutableSet::of, shardManager, true); - ConnectorSplitSource splitSource = getSplits(raptorSplitManagerWithBackup, tableHandle); - getSplits(splitSource, 1000); - }) - .isInstanceOf(TrinoException.class) - .hasMessage("No nodes available to run query"); - } - - private void deleteShardNodes() - { - dummyHandle.execute("DELETE FROM shard_nodes"); - dummyHandle.execute(format("UPDATE %s SET node_ids = ''", shardIndexTable(tableId))); - } - - private static ConnectorSplitSource getSplits(RaptorSplitManager splitManager, ConnectorTableHandle table) - { - ConnectorTransactionHandle transaction = new RaptorTransactionHandle(); - return splitManager.getSplits(transaction, SESSION, table, DynamicFilter.EMPTY, Constraint.alwaysTrue()); - } - - private static List getSplits(ConnectorSplitSource source, int maxSize) - { - return getFutureValue(source.getNextBatch(maxSize)).getSplits(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleaner.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleaner.java deleted file mode 100644 index 8e174bc1b535..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleaner.java +++ /dev/null @@ -1,464 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableSet; -import io.airlift.testing.TestingTicker; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.backup.FileBackupStore; -import io.trino.plugin.raptor.legacy.storage.FileStorageService; -import io.trino.plugin.raptor.legacy.storage.StorageService; -import io.trino.plugin.raptor.legacy.util.DaoSupplier; -import io.trino.plugin.raptor.legacy.util.UuidUtil.UuidArgumentFactory; -import org.intellij.lang.annotations.Language; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.sqlobject.config.RegisterArgumentFactory; -import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidFromBytes; -import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder; -import static java.nio.file.Files.createTempDirectory; -import static java.util.Arrays.asList; -import static java.util.UUID.randomUUID; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardCleaner -{ - private Jdbi dbi; - private Handle dummyHandle; - private Path temporary; - private StorageService storageService; - private BackupStore backupStore; - private TestingTicker ticker; - private ShardCleaner cleaner; - - @BeforeEach - public void setup() - throws IOException - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - - temporary = createTempDirectory(null); - File directory = temporary.resolve("data").toFile(); - storageService = new FileStorageService(directory); - storageService.start(); - - File backupDirectory = temporary.resolve("backup").toFile(); - backupStore = new FileBackupStore(backupDirectory); - ((FileBackupStore) backupStore).start(); - - ticker = new TestingTicker(); - - ShardCleanerConfig config = new ShardCleanerConfig(); - cleaner = new ShardCleaner( - new DaoSupplier<>(dbi, H2ShardDao.class), - "node1", - true, - ticker, - storageService, - Optional.of(backupStore), - config.getMaxTransactionAge(), - config.getTransactionCleanerInterval(), - config.getLocalCleanerInterval(), - config.getLocalCleanTime(), - config.getBackupCleanerInterval(), - config.getBackupCleanTime(), - config.getBackupDeletionThreads(), - config.getMaxCompletedTransactionAge()); - } - - @AfterEach - public void teardown() - throws IOException - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testAbortOldTransactions() - { - TestingDao dao = dbi.onDemand(TestingDao.class); - - long now = System.currentTimeMillis(); - - long txn1 = dao.insertTransaction(new Timestamp(now - HOURS.toMillis(26))); - long txn2 = dao.insertTransaction(new Timestamp(now - HOURS.toMillis(25))); - long txn3 = dao.insertTransaction(new Timestamp(now)); - - ShardDao shardDao = dbi.onDemand(ShardDao.class); - assertThat(shardDao.finalizeTransaction(txn1, true)).isEqualTo(1); - - assertQuery("SELECT transaction_id, successful FROM transactions", - row(txn1, true), - row(txn2, null), - row(txn3, null)); - - cleaner.abortOldTransactions(); - - assertQuery("SELECT transaction_id, successful FROM transactions", - row(txn1, true), - row(txn2, false), - row(txn3, null)); - } - - @Test - public void testDeleteOldShards() - { - assertThat(cleaner.getBackupShardsQueued().getTotalCount()).isEqualTo(0); - - ShardDao dao = dbi.onDemand(ShardDao.class); - - UUID shard1 = randomUUID(); - UUID shard2 = randomUUID(); - UUID shard3 = randomUUID(); - - // shards for failed transaction - long txn1 = dao.insertTransaction(); - assertThat(dao.finalizeTransaction(txn1, false)).isEqualTo(1); - - dao.insertCreatedShard(shard1, txn1); - dao.insertCreatedShard(shard2, txn1); - - // shards for running transaction - long txn2 = dao.insertTransaction(); - - dao.insertCreatedShard(shard3, txn2); - - // verify database - assertQuery("SELECT shard_uuid, transaction_id FROM created_shards", - row(shard1, txn1), - row(shard2, txn1), - row(shard3, txn2)); - - assertQuery("SELECT shard_uuid FROM deleted_shards"); - - // move shards for failed transaction to deleted - cleaner.deleteOldShards(); - - assertThat(cleaner.getBackupShardsQueued().getTotalCount()).isEqualTo(2); - - // verify database - assertQuery("SELECT shard_uuid, transaction_id FROM created_shards", - row(shard3, txn2)); - - assertQuery("SELECT shard_uuid FROM deleted_shards", - row(shard1), - row(shard2)); - } - - @Test - public void testCleanLocalShardsImmediately() - throws Exception - { - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(0); - TestingShardDao shardDao = dbi.onDemand(TestingShardDao.class); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - long tableId = metadataDao.insertTable("test", "test", false, false, null, 0); - - UUID shard1 = randomUUID(); - UUID shard2 = randomUUID(); - UUID shard3 = randomUUID(); - - Set shards = ImmutableSet.of(shard1, shard2, shard3); - - for (UUID shard : shards) { - shardDao.insertShard(shard, tableId, null, 0, 0, 0, 0); - createShardFile(shard); - assertThat(shardFileExists(shard)).isTrue(); - } - - int node1 = shardDao.insertNode("node1"); - int node2 = shardDao.insertNode("node2"); - - // shard 1: referenced by this node - // shard 2: not referenced - // shard 3: referenced by other node - - shardDao.insertShardNode(shard1, node1); - shardDao.insertShardNode(shard3, node2); - - // clean shards immediately - Set local = cleaner.getLocalShards(); - cleaner.cleanLocalShardsImmediately(local); - - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(2); - - // shards 2 and 3 should be deleted - // shard 1 is referenced by this node - assertThat(shardFileExists(shard1)).isTrue(); - assertThat(shardFileExists(shard2)).isFalse(); - assertThat(shardFileExists(shard3)).isFalse(); - } - - @Test - public void testCleanLocalShards() - throws Exception - { - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(0); - - TestingShardDao shardDao = dbi.onDemand(TestingShardDao.class); - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - long tableId = metadataDao.insertTable("test", "test", false, false, null, 0); - - UUID shard1 = randomUUID(); - UUID shard2 = randomUUID(); - UUID shard3 = randomUUID(); - UUID shard4 = randomUUID(); - - Set shards = ImmutableSet.of(shard1, shard2, shard3, shard4); - - for (UUID shard : shards) { - shardDao.insertShard(shard, tableId, null, 0, 0, 0, 0); - createShardFile(shard); - assertThat(shardFileExists(shard)).isTrue(); - } - - int node1 = shardDao.insertNode("node1"); - int node2 = shardDao.insertNode("node2"); - - // shard 1: referenced by this node - // shard 2: not referenced - // shard 3: not referenced - // shard 4: referenced by other node - - shardDao.insertShardNode(shard1, node1); - shardDao.insertShardNode(shard4, node2); - - // mark unreferenced shards - cleaner.cleanLocalShards(); - - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(0); - - // make sure nothing is deleted - for (UUID shard : shards) { - assertThat(shardFileExists(shard)).isTrue(); - } - - // add reference for shard 3 - shardDao.insertShardNode(shard3, node1); - - // advance time beyond clean time - Duration cleanTime = new ShardCleanerConfig().getLocalCleanTime(); - ticker.increment(cleanTime.toMillis() + 1, MILLISECONDS); - - // clean shards - cleaner.cleanLocalShards(); - - assertThat(cleaner.getLocalShardsCleaned().getTotalCount()).isEqualTo(2); - - // shards 2 and 4 should be deleted - // shards 1 and 3 are referenced by this node - assertThat(shardFileExists(shard1)).isTrue(); - assertThat(shardFileExists(shard2)).isFalse(); - assertThat(shardFileExists(shard3)).isTrue(); - assertThat(shardFileExists(shard4)).isFalse(); - } - - @Test - public void testCleanBackupShards() - throws Exception - { - assertThat(cleaner.getBackupShardsCleaned().getTotalCount()).isEqualTo(0); - - TestingDao dao = dbi.onDemand(TestingDao.class); - - UUID shard1 = randomUUID(); - UUID shard2 = randomUUID(); - UUID shard3 = randomUUID(); - - long now = System.currentTimeMillis(); - Timestamp time1 = new Timestamp(now - HOURS.toMillis(25)); - Timestamp time2 = new Timestamp(now - HOURS.toMillis(23)); - - // shard 1: should be cleaned - dao.insertDeletedShard(shard1, time1); - - // shard 2: should be cleaned - dao.insertDeletedShard(shard2, time1); - - // shard 3: deleted too recently - dao.insertDeletedShard(shard3, time2); - - createShardBackups(shard1, shard2, shard3); - - cleaner.cleanBackupShards(); - - assertThat(cleaner.getBackupShardsCleaned().getTotalCount()).isEqualTo(2); - - assertThat(shardBackupExists(shard1)).isFalse(); - assertThat(shardBackupExists(shard2)).isFalse(); - assertThat(shardBackupExists(shard3)).isTrue(); - - assertQuery("SELECT shard_uuid FROM deleted_shards", - row(shard3)); - } - - @Test - public void testDeleteOldCompletedTransactions() - { - TestingDao dao = dbi.onDemand(TestingDao.class); - ShardDao shardDao = dbi.onDemand(ShardDao.class); - - long now = System.currentTimeMillis(); - Timestamp yesterdayStart = new Timestamp(now - HOURS.toMillis(27)); - Timestamp yesterdayEnd = new Timestamp(now - HOURS.toMillis(26)); - Timestamp todayEnd = new Timestamp(now - HOURS.toMillis(1)); - - long txn1 = dao.insertTransaction(yesterdayStart); - long txn2 = dao.insertTransaction(yesterdayStart); - long txn3 = dao.insertTransaction(yesterdayStart); - long txn4 = dao.insertTransaction(yesterdayStart); - long txn5 = dao.insertTransaction(new Timestamp(now)); - long txn6 = dao.insertTransaction(new Timestamp(now)); - - assertThat(shardDao.finalizeTransaction(txn1, true)).isEqualTo(1); - assertThat(shardDao.finalizeTransaction(txn2, false)).isEqualTo(1); - assertThat(shardDao.finalizeTransaction(txn3, false)).isEqualTo(1); - assertThat(shardDao.finalizeTransaction(txn5, true)).isEqualTo(1); - assertThat(shardDao.finalizeTransaction(txn6, false)).isEqualTo(1); - - assertThat(dao.updateTransactionEndTime(txn1, yesterdayEnd)).isEqualTo(1); - assertThat(dao.updateTransactionEndTime(txn2, yesterdayEnd)).isEqualTo(1); - assertThat(dao.updateTransactionEndTime(txn3, yesterdayEnd)).isEqualTo(1); - assertThat(dao.updateTransactionEndTime(txn5, todayEnd)).isEqualTo(1); - assertThat(dao.updateTransactionEndTime(txn6, todayEnd)).isEqualTo(1); - - shardDao.insertCreatedShard(randomUUID(), txn2); - shardDao.insertCreatedShard(randomUUID(), txn2); - - assertQuery("SELECT transaction_id, successful, end_time FROM transactions", - row(txn1, true, yesterdayEnd), // old successful - row(txn2, false, yesterdayEnd), // old failed, shards present - row(txn3, false, yesterdayEnd), // old failed, no referencing shards - row(txn4, null, null), // old not finished - row(txn5, true, todayEnd), // new successful - row(txn6, false, todayEnd)); // new failed, no referencing shards - - cleaner.deleteOldCompletedTransactions(); - - assertQuery("SELECT transaction_id, successful, end_time FROM transactions", - row(txn2, false, yesterdayEnd), - row(txn4, null, null), - row(txn5, true, todayEnd), - row(txn6, false, todayEnd)); - } - - private boolean shardFileExists(UUID uuid) - { - return storageService.getStorageFile(uuid).exists(); - } - - private void createShardFile(UUID uuid) - throws IOException - { - File file = storageService.getStorageFile(uuid); - storageService.createParents(file); - assertThat(file.createNewFile()).isTrue(); - } - - private boolean shardBackupExists(UUID uuid) - { - return backupStore.shardExists(uuid); - } - - private void createShardBackups(UUID... uuids) - throws IOException - { - for (UUID uuid : uuids) { - File file = temporary.resolve("empty-" + randomUUID()).toFile(); - assertThat(file.createNewFile()).isTrue(); - backupStore.backupShard(uuid, file); - } - } - - @SafeVarargs - private void assertQuery(@Language("SQL") String sql, List... rows) - { - assertEqualsIgnoreOrder(select(sql), asList(rows)); - } - - private List> select(@Language("SQL") String sql) - { - return dbi.withHandle(handle -> handle.createQuery(sql) - .map((rs, index, context) -> { - int count = rs.getMetaData().getColumnCount(); - List row = new ArrayList<>(count); - for (int i = 1; i <= count; i++) { - Object value = rs.getObject(i); - if (value instanceof byte[]) { - value = uuidFromBytes((byte[]) value); - } - row.add(value); - } - return row; - }) - .list()); - } - - private static List row(Object... values) - { - return asList(values); - } - - @RegisterArgumentFactory(UuidArgumentFactory.class) - private interface TestingDao - { - @SqlUpdate("INSERT INTO transactions (start_time) VALUES (:startTime)") - @GetGeneratedKeys - long insertTransaction(Timestamp startTime); - - @SqlUpdate("INSERT INTO deleted_shards (shard_uuid, delete_time)\n" + - "VALUES (:shardUuid, :deleteTime)") - void insertDeletedShard(UUID shardUuid, Timestamp deleteTime); - - @SqlUpdate("UPDATE transactions SET end_time = :endTime WHERE transaction_id = :transactionId") - int updateTransactionEndTime(long transactionId, Timestamp endTime); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleanerConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleanerConfig.java deleted file mode 100644 index 92fff624f373..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardCleanerConfig.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MINUTES; - -public class TestShardCleanerConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(ShardCleanerConfig.class) - .setMaxTransactionAge(new Duration(1, DAYS)) - .setTransactionCleanerInterval(new Duration(10, MINUTES)) - .setLocalCleanerInterval(new Duration(1, HOURS)) - .setLocalCleanTime(new Duration(4, HOURS)) - .setBackupCleanerInterval(new Duration(5, MINUTES)) - .setBackupCleanTime(new Duration(1, DAYS)) - .setBackupDeletionThreads(50) - .setMaxCompletedTransactionAge(new Duration(1, DAYS))); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("raptor.max-transaction-age", "42m") - .put("raptor.transaction-cleaner-interval", "43m") - .put("raptor.local-cleaner-interval", "31m") - .put("raptor.local-clean-time", "32m") - .put("raptor.backup-cleaner-interval", "34m") - .put("raptor.backup-clean-time", "35m") - .put("raptor.backup-deletion-threads", "37") - .put("raptor.max-completed-transaction-age", "39m") - .buildOrThrow(); - - ShardCleanerConfig expected = new ShardCleanerConfig() - .setMaxTransactionAge(new Duration(42, MINUTES)) - .setTransactionCleanerInterval(new Duration(43, MINUTES)) - .setLocalCleanerInterval(new Duration(31, MINUTES)) - .setLocalCleanTime(new Duration(32, MINUTES)) - .setBackupCleanerInterval(new Duration(34, MINUTES)) - .setBackupCleanTime(new Duration(35, MINUTES)) - .setBackupDeletionThreads(37) - .setMaxCompletedTransactionAge(new Duration(39, MINUTES)); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardDao.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardDao.java deleted file mode 100644 index 09751ea27780..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardDao.java +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.statement.UnableToExecuteStatementException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.sql.SQLException; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardDao -{ - private TestingShardDao dao; - private Jdbi dbi; - private Handle dummyHandle; - - @BeforeEach - public void setup() - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - dao = dbi.onDemand(TestingShardDao.class); - createTablesWithRetry(dbi); - } - - @AfterEach - public void teardown() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testExternalBatches() - { - assertThat(dao.externalBatchExists("foo")).isFalse(); - assertThat(dao.externalBatchExists("bar")).isFalse(); - - dao.insertExternalBatch("foo"); - - assertThat(dao.externalBatchExists("foo")).isTrue(); - assertThat(dao.externalBatchExists("bar")).isFalse(); - - assertThatThrownBy(() -> dao.insertExternalBatch("foo")) - .isInstanceOfSatisfying(UnableToExecuteStatementException.class, e -> { - assertInstanceOf(e.getCause(), SQLException.class); - assertThat(((SQLException) e.getCause()).getSQLState()).startsWith("23"); - }); - } - - @Test - public void testInsertCreatedShard() - { - long transactionId = dao.insertTransaction(); - dao.insertCreatedShard(UUID.randomUUID(), transactionId); - dao.deleteCreatedShards(transactionId); - } - - @Test - public void testInsertDeletedShards() - { - dao.insertDeletedShards(ImmutableList.of(UUID.randomUUID(), UUID.randomUUID())); - dao.insertDeletedShards(0); - } - - @Test - public void testNodeInsert() - { - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of()); - - String nodeName = UUID.randomUUID().toString(); - int nodeId = dao.insertNode(nodeName); - assertThat(dao.getNodeId(nodeName)).isEqualTo((Integer) nodeId); - - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of(nodeName)); - } - - @Test - public void testInsertShard() - { - long tableId = createTable("test"); - UUID shardUuid = UUID.randomUUID(); - long shardId = dao.insertShard(shardUuid, tableId, null, 13, 42, 84, 1234); - - ShardMetadata shard = dao.getShard(shardUuid); - assertThat(shard).isNotNull(); - assertThat(shard.getTableId()).isEqualTo(tableId); - assertThat(shard.getShardId()).isEqualTo(shardId); - assertThat(shard.getShardUuid()).isEqualTo(shardUuid); - assertThat(shard.getRowCount()).isEqualTo(13); - assertThat(shard.getCompressedSize()).isEqualTo(42); - assertThat(shard.getUncompressedSize()).isEqualTo(84); - assertThat(shard.getXxhash64()).isEqualTo(OptionalLong.of(1234)); - } - - @Test - public void testInsertShardNodeUsingShardUuid() - { - int nodeId = dao.insertNode("node"); - - long tableId = createTable("test"); - UUID shard = UUID.randomUUID(); - dao.insertShard(shard, tableId, null, 0, 0, 0, 0); - - dao.insertShardNode(shard, nodeId); - - assertThat(dao.getShardNodes(tableId)).containsExactlyElementsOf(ImmutableList.of(new ShardNode(shard, "node"))); - } - - @Test - public void testNodeShards() - { - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of()); - - String nodeName1 = UUID.randomUUID().toString(); - int nodeId1 = dao.insertNode(nodeName1); - - String nodeName2 = UUID.randomUUID().toString(); - int nodeId2 = dao.insertNode(nodeName2); - - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of(nodeName1, nodeName2)); - - UUID shardUuid1 = UUID.randomUUID(); - UUID shardUuid2 = UUID.randomUUID(); - UUID shardUuid3 = UUID.randomUUID(); - UUID shardUuid4 = UUID.randomUUID(); - UUID shardUuid5 = UUID.randomUUID(); - - MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - - int bucketCount = 20; - long distributionId = metadataDao.insertDistribution("test", "bigint", bucketCount); - for (int i = 0; i < bucketCount; i++) { - Integer nodeId = ((i % 2) == 0) ? nodeId1 : nodeId2; - dao.insertBuckets(distributionId, ImmutableList.of(i), ImmutableList.of(nodeId)); - } - - long plainTableId = metadataDao.insertTable("test", "plain", false, false, null, 0); - long bucketedTableId = metadataDao.insertTable("test", "bucketed", false, false, distributionId, 0); - - long shardId1 = dao.insertShard(shardUuid1, plainTableId, null, 1, 11, 111, 888_111); - long shardId2 = dao.insertShard(shardUuid2, plainTableId, null, 2, 22, 222, 888_222); - long shardId3 = dao.insertShard(shardUuid3, bucketedTableId, 8, 3, 33, 333, 888_333); - long shardId4 = dao.insertShard(shardUuid4, bucketedTableId, 9, 4, 44, 444, 888_444); - long shardId5 = dao.insertShard(shardUuid5, bucketedTableId, 7, 5, 55, 555, 888_555); - - OptionalInt noBucket = OptionalInt.empty(); - OptionalLong noRange = OptionalLong.empty(); - ShardMetadata shard1 = new ShardMetadata(plainTableId, shardId1, shardUuid1, noBucket, 1, 11, 111, OptionalLong.of(888_111), noRange, noRange); - ShardMetadata shard2 = new ShardMetadata(plainTableId, shardId2, shardUuid2, noBucket, 2, 22, 222, OptionalLong.of(888_222), noRange, noRange); - ShardMetadata shard3 = new ShardMetadata(bucketedTableId, shardId3, shardUuid3, OptionalInt.of(8), 3, 33, 333, OptionalLong.of(888_333), noRange, noRange); - ShardMetadata shard4 = new ShardMetadata(bucketedTableId, shardId4, shardUuid4, OptionalInt.of(9), 4, 44, 444, OptionalLong.of(888_444), noRange, noRange); - ShardMetadata shard5 = new ShardMetadata(bucketedTableId, shardId5, shardUuid5, OptionalInt.of(7), 5, 55, 555, OptionalLong.of(888_555), noRange, noRange); - - assertThat(dao.getShards(plainTableId)).isEqualTo(ImmutableSet.of(shardUuid1, shardUuid2)); - assertThat(dao.getShards(bucketedTableId)).isEqualTo(ImmutableSet.of(shardUuid3, shardUuid4, shardUuid5)); - - assertThat(dao.getNodeShards(nodeName1, null)).isEqualTo(ImmutableSet.of(shard3)); - assertThat(dao.getNodeShards(nodeName2, null)).isEqualTo(ImmutableSet.of(shard4, shard5)); - assertThat(dao.getNodeSizes()).isEqualTo(ImmutableSet.of( - new NodeSize(nodeName1, 33), - new NodeSize(nodeName2, 44 + 55))); - - dao.insertShardNode(shardId1, nodeId1); - dao.insertShardNode(shardId2, nodeId1); - dao.insertShardNode(shardId1, nodeId2); - - assertThat(dao.getNodeShards(nodeName1, null)).isEqualTo(ImmutableSet.of(shard1, shard2, shard3)); - assertThat(dao.getNodeShards(nodeName2, null)).isEqualTo(ImmutableSet.of(shard1, shard4, shard5)); - assertThat(dao.getNodeSizes()).isEqualTo(ImmutableSet.of( - new NodeSize(nodeName1, 11 + 22 + 33), - new NodeSize(nodeName2, 11 + 44 + 55))); - - dao.dropShardNodes(plainTableId); - - assertThat(dao.getNodeShards(nodeName1, null)).isEqualTo(ImmutableSet.of(shard3)); - assertThat(dao.getNodeShards(nodeName2, null)).isEqualTo(ImmutableSet.of(shard4, shard5)); - assertThat(dao.getNodeSizes()).isEqualTo(ImmutableSet.of( - new NodeSize(nodeName1, 33), - new NodeSize(nodeName2, 44 + 55))); - - dao.dropShards(plainTableId); - dao.dropShards(bucketedTableId); - - assertThat(dao.getShards(plainTableId)).isEqualTo(ImmutableSet.of()); - assertThat(dao.getShards(bucketedTableId)).isEqualTo(ImmutableSet.of()); - assertThat(dao.getNodeSizes()).isEqualTo(ImmutableSet.of()); - } - - @Test - public void testShardSelection() - { - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of()); - - String nodeName1 = UUID.randomUUID().toString(); - int nodeId1 = dao.insertNode(nodeName1); - - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of(nodeName1)); - - String nodeName2 = UUID.randomUUID().toString(); - int nodeId2 = dao.insertNode(nodeName2); - - assertThat(dao.getAllNodesInUse()).isEqualTo(ImmutableSet.of(nodeName1, nodeName2)); - - long tableId = createTable("test"); - - UUID shardUuid1 = UUID.randomUUID(); - UUID shardUuid2 = UUID.randomUUID(); - UUID shardUuid3 = UUID.randomUUID(); - UUID shardUuid4 = UUID.randomUUID(); - - long shardId1 = dao.insertShard(shardUuid1, tableId, null, 0, 0, 0, 0); - long shardId2 = dao.insertShard(shardUuid2, tableId, null, 0, 0, 0, 0); - long shardId3 = dao.insertShard(shardUuid3, tableId, null, 0, 0, 0, 0); - long shardId4 = dao.insertShard(shardUuid4, tableId, null, 0, 0, 0, 0); - - Set shards = dao.getShards(tableId); - assertThat(shards.size()).isEqualTo(4); - - assertThat(shards).contains(shardUuid1); - assertThat(shards).contains(shardUuid2); - assertThat(shards).contains(shardUuid3); - assertThat(shards).contains(shardUuid4); - - assertThat(dao.getShardNodes(tableId).size()).isEqualTo(0); - - dao.insertShardNode(shardId1, nodeId1); - dao.insertShardNode(shardId1, nodeId2); - dao.insertShardNode(shardId2, nodeId1); - dao.insertShardNode(shardId3, nodeId1); - dao.insertShardNode(shardId4, nodeId1); - dao.insertShardNode(shardId4, nodeId2); - - assertThat(dao.getShards(tableId)).isEqualTo(shards); - - Set shardNodes = dao.getShardNodes(tableId); - assertThat(shardNodes.size()).isEqualTo(6); - - assertContainsShardNode(shardNodes, nodeName1, shardUuid1); - assertContainsShardNode(shardNodes, nodeName2, shardUuid1); - assertContainsShardNode(shardNodes, nodeName1, shardUuid2); - assertContainsShardNode(shardNodes, nodeName1, shardUuid3); - assertContainsShardNode(shardNodes, nodeName1, shardUuid4); - assertContainsShardNode(shardNodes, nodeName2, shardUuid4); - } - - private long createTable(String name) - { - return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0); - } - - private static void assertContainsShardNode(Set nodes, String nodeName, UUID shardUuid) - { - assertThat(nodes).contains(new ShardNode(shardUuid, nodeName)); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardPredicate.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardPredicate.java deleted file mode 100644 index c7cc79e6cac9..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestShardPredicate.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.spi.predicate.SortedRangeSet; -import io.trino.spi.predicate.TupleDomain; -import org.junit.jupiter.api.Test; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.bucketNumberColumnHandle; -import static io.trino.plugin.raptor.legacy.RaptorColumnHandle.shardUuidColumnHandle; -import static io.trino.plugin.raptor.legacy.util.UuidUtil.uuidStringToBytes; -import static io.trino.spi.predicate.Domain.create; -import static io.trino.spi.predicate.Domain.singleValue; -import static io.trino.spi.predicate.Range.equal; -import static io.trino.spi.predicate.Range.greaterThanOrEqual; -import static io.trino.spi.predicate.TupleDomain.withColumnDomains; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.sql.JDBCType.VARBINARY; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestShardPredicate -{ - @Test - public void testSimpleShardUuidPredicate() - { - String uuid = randomUUID().toString(); - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - shardUuidColumnHandle(), singleValue(VARCHAR, utf8Slice(uuid)))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - - assertThat(shardPredicate.getPredicate()).isEqualTo("shard_uuid = ?"); - assertThat(shardPredicate.getTypes()).isEqualTo(ImmutableList.of(VARBINARY)); - assertThat(shardPredicate.getValues()).isEqualTo(ImmutableList.of(uuidStringToBytes(utf8Slice(uuid)))); - } - - @Test - public void testDiscreteShardUuidPredicate() - { - Slice uuid0 = utf8Slice(randomUUID().toString()); - Slice uuid1 = utf8Slice(randomUUID().toString()); - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - shardUuidColumnHandle(), - create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(equal(VARCHAR, uuid0), equal(VARCHAR, uuid1))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - - assertThat(shardPredicate.getPredicate()).isEqualTo("shard_uuid = ? OR shard_uuid = ?"); - assertThat(shardPredicate.getTypes()).isEqualTo(ImmutableList.of(VARBINARY, VARBINARY)); - assertThat(ImmutableSet.copyOf(shardPredicate.getValues())).isEqualTo(ImmutableSet.of(uuidStringToBytes(uuid0), uuidStringToBytes(uuid1))); - } - - @Test - public void testInvalidUuid() - { - Slice uuid0 = utf8Slice("test1"); - Slice uuid1 = utf8Slice("test2"); - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - shardUuidColumnHandle(), - create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(equal(VARCHAR, uuid0), equal(VARCHAR, uuid1))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - - assertThat(shardPredicate.getPredicate()).isEqualTo("true"); - } - - @Test - public void testRangeShardUuidPredicate() - { - Slice uuid0 = utf8Slice(randomUUID().toString()); - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - shardUuidColumnHandle(), - create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(greaterThanOrEqual(VARCHAR, uuid0))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - assertThat(shardPredicate.getPredicate()).isEqualTo("true"); - } - - @Test - public void testBucketNumberSingleRange() - { - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - bucketNumberColumnHandle(), - create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - assertThat(shardPredicate.getPredicate()).isEqualTo("(((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)))"); - } - - @Test - public void testBucketNumberMultipleRanges() - { - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - bucketNumberColumnHandle(), - create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false))); - - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - assertThat(shardPredicate.getPredicate()).isEqualTo("(((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL))" + - " OR ((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)))"); - } - - @Test - public void testMultipleColumnsMultipleRanges() - { - TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( - bucketNumberColumnHandle(), - create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false), - new RaptorColumnHandle("col", 1, INTEGER), - create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false))); - ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); - assertThat(shardPredicate.getPredicate()).isEqualTo("(((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)) " + - "OR ((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL))) " + - "AND (((c1_max >= ? OR c1_max IS NULL) AND (c1_min <= ? OR c1_min IS NULL)) " + - "OR ((c1_max >= ? OR c1_max IS NULL) AND (c1_min <= ? OR c1_min IS NULL)))"); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestingShardDao.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestingShardDao.java deleted file mode 100644 index c402797352b6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/metadata/TestingShardDao.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.metadata; - -import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; -import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; -import org.jdbi.v3.sqlobject.statement.SqlQuery; -import org.jdbi.v3.sqlobject.statement.SqlUpdate; - -import java.util.Set; -import java.util.UUID; - -@RegisterConstructorMapper(ShardNode.class) -interface TestingShardDao - extends H2ShardDao -{ - @SqlQuery("SELECT shard_uuid FROM shards WHERE table_id = :tableId") - Set getShards(long tableId); - - @SqlQuery("SELECT s.shard_uuid, n.node_identifier\n" + - "FROM shards s\n" + - "JOIN shard_nodes sn ON (s.shard_id = sn.shard_id)\n" + - "JOIN nodes n ON (sn.node_id = n.node_id)\n" + - "WHERE s.table_id = :tableId") - Set getShardNodes(long tableId); - - @SqlQuery("SELECT node_identifier FROM nodes") - Set getAllNodesInUse(); - - @SqlUpdate("INSERT INTO shards (shard_uuid, table_id, bucket_number, create_time, row_count, compressed_size, uncompressed_size, xxhash64)\n" + - "VALUES (:shardUuid, :tableId, :bucketNumber, CURRENT_TIMESTAMP, :rowCount, :compressedSize, :uncompressedSize, :xxhash64)") - @GetGeneratedKeys - long insertShard( - UUID shardUuid, - long tableId, - Integer bucketNumber, - long rowCount, - long compressedSize, - long uncompressedSize, - long xxhash64); - - @SqlUpdate("INSERT INTO shard_nodes (shard_id, node_id)\n" + - "VALUES (:shardId, :nodeId)\n") - void insertShardNode(long shardId, int nodeId); -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorFileBasedSecurity.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorFileBasedSecurity.java deleted file mode 100644 index dade69191d45..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorFileBasedSecurity.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import com.google.common.io.Resources; -import io.trino.Session; -import io.trino.plugin.raptor.legacy.RaptorQueryRunner; -import io.trino.spi.security.Identity; -import io.trino.testing.QueryRunner; -import io.trino.tpch.TpchTable; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; - -import static io.trino.testing.TestingSession.testSessionBuilder; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; - -@TestInstance(PER_CLASS) -@Execution(CONCURRENT) -public class TestRaptorFileBasedSecurity -{ - private final QueryRunner queryRunner; - - public TestRaptorFileBasedSecurity() - throws Exception - { - String path = new File(Resources.getResource(getClass(), "security.json").toURI()).getPath(); - queryRunner = RaptorQueryRunner.builder() - .addConnectorProperty("security.config-file", path) - .addConnectorProperty("raptor.security", "file") - .setInitialTables(TpchTable.getTables()) - .build(); - } - - @AfterAll - public void tearDown() - { - queryRunner.close(); - } - - @Test - public void testAdminCanRead() - { - Session admin = getSession("user"); - queryRunner.execute(admin, "SELECT * FROM orders"); - } - - @Test - public void testNonAdminCannotRead() - { - assertThatThrownBy(() -> { - Session bob = getSession("bob"); - queryRunner.execute(bob, "SELECT * FROM orders"); - }) - .isInstanceOf(RuntimeException.class) - .hasMessageMatching(".*Access Denied: Cannot select from table tpch.orders.*"); - } - - private Session getSession(String user) - { - return testSessionBuilder() - .setCatalog(queryRunner.getDefaultSession().getCatalog()) - .setSchema(queryRunner.getDefaultSession().getSchema()) - .setIdentity(Identity.ofUser(user)) - .build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorReadOnlySecurity.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorReadOnlySecurity.java deleted file mode 100644 index 65e647a8ba40..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorReadOnlySecurity.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import io.trino.plugin.raptor.legacy.RaptorQueryRunner; -import io.trino.testing.QueryRunner; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class TestRaptorReadOnlySecurity -{ - @Test - public void testCannotWrite() - throws Exception - { - try (QueryRunner queryRunner = RaptorQueryRunner.builder().addConnectorProperty("raptor.security", "read-only").build()) { - assertThatThrownBy(() -> { - queryRunner.execute("CREATE TABLE test_create (a bigint, b double, c varchar)"); - }) - .isInstanceOf(RuntimeException.class) - .hasMessageMatching(".*Access Denied: Cannot create .*"); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorSecurityConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorSecurityConfig.java deleted file mode 100644 index 449067454b2f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/security/TestRaptorSecurityConfig.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.security; - -import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static io.trino.plugin.raptor.legacy.security.RaptorSecurity.ALLOW_ALL; -import static io.trino.plugin.raptor.legacy.security.RaptorSecurity.READ_ONLY; - -public class TestRaptorSecurityConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(RaptorSecurityConfig.class) - .setSecuritySystem(ALLOW_ALL)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.of("raptor.security", "read-only"); - - RaptorSecurityConfig expected = new RaptorSecurityConfig() - .setSecuritySystem(READ_ONLY); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/InMemoryShardRecorder.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/InMemoryShardRecorder.java deleted file mode 100644 index 85fac0528dcf..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/InMemoryShardRecorder.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -import static java.util.Objects.requireNonNull; - -public class InMemoryShardRecorder - implements ShardRecorder -{ - private final List shards = new ArrayList<>(); - - public List getShards() - { - return ImmutableList.copyOf(shards); - } - - @Override - public void recordCreatedShard(long transactionId, UUID shardUuid) - { - shards.add(new RecordedShard(transactionId, shardUuid)); - } - - public static class RecordedShard - { - private final long transactionId; - private final UUID shardUuid; - - public RecordedShard(long transactionId, UUID shardUuid) - { - this.transactionId = transactionId; - this.shardUuid = requireNonNull(shardUuid, "shardUuid is null"); - } - - public long getTransactionId() - { - return transactionId; - } - - public UUID getShardUuid() - { - return shardUuid; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/OrcTestingUtil.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/OrcTestingUtil.java deleted file mode 100644 index 8a209cadd2d6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/OrcTestingUtil.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.primitives.UnsignedBytes; -import io.airlift.units.DataSize; -import io.trino.orc.FileOrcDataSource; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcPredicate; -import io.trino.orc.OrcReader; -import io.trino.orc.OrcReaderOptions; -import io.trino.orc.OrcRecordReader; -import io.trino.spi.type.Type; -import org.joda.time.DateTimeZone; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.List; - -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static io.trino.orc.OrcReader.MAX_BATCH_SIZE; -import static org.assertj.core.api.Assertions.assertThat; - -final class OrcTestingUtil -{ - private OrcTestingUtil() {} - - private static final OrcReaderOptions READER_OPTIONS = new OrcReaderOptions() - .withMaxReadBlockSize(DataSize.of(1, MEGABYTE)) - .withMaxMergeDistance(DataSize.of(1, MEGABYTE)) - .withMaxBufferSize(DataSize.of(1, MEGABYTE)) - .withStreamBufferSize(DataSize.of(1, MEGABYTE)) - .withTinyStripeThreshold(DataSize.of(1, MEGABYTE)); - - public static OrcDataSource fileOrcDataSource(File file) - throws FileNotFoundException - { - return new FileOrcDataSource(file, READER_OPTIONS); - } - - public static OrcRecordReader createReader(OrcDataSource dataSource, List columnIds, List types) - throws IOException - { - OrcReader orcReader = OrcReader.createOrcReader(dataSource, READER_OPTIONS) - .orElseThrow(() -> new RuntimeException("File is empty")); - - List columnNames = orcReader.getColumnNames(); - assertThat(columnNames.size()).isEqualTo(columnIds.size()); - - return orcReader.createRecordReader( - orcReader.getRootColumn().getNestedColumns(), - types, - OrcPredicate.TRUE, - DateTimeZone.UTC, - newSimpleAggregatedMemoryContext(), - MAX_BATCH_SIZE, - RuntimeException::new); - } - - public static byte[] octets(int... values) - { - byte[] bytes = new byte[values.length]; - for (int i = 0; i < bytes.length; i++) { - bytes[i] = UnsignedBytes.checkedCast(values[i]); - } - return bytes; - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancer.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancer.java deleted file mode 100644 index ab1bc789e346..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancer.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.HashMultiset; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Multiset; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.client.NodeVersion; -import io.trino.metadata.InternalNode; -import io.trino.plugin.raptor.legacy.NodeSupplier; -import io.trino.plugin.raptor.legacy.metadata.BucketNode; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.Distribution; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.storage.BucketBalancer.BucketAssignment; -import io.trino.plugin.raptor.legacy.storage.BucketBalancer.ClusterState; -import io.trino.spi.Node; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.net.URI; -import java.util.List; -import java.util.OptionalLong; -import java.util.stream.Collectors; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertLessThanOrEqual; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.Distribution.serializeColumnTypes; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.spi.type.BigintType.BIGINT; -import static java.util.concurrent.TimeUnit.DAYS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestBucketBalancer -{ - private static final List AVAILABLE_WORKERS = ImmutableList.of("node1", "node2", "node3", "node4", "node5"); - - private Jdbi dbi; - private Handle dummyHandle; - private ShardManager shardManager; - private TestingNodeManager nodeManager; - private MetadataDao metadataDao; - private BucketBalancer balancer; - - @BeforeEach - public void setup() - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - - metadataDao = dbi.onDemand(MetadataDao.class); - nodeManager = new TestingNodeManager(AVAILABLE_WORKERS.stream() - .map(TestBucketBalancer::createTestingNode) - .collect(Collectors.toList())); - - NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - shardManager = createShardManager(dbi, nodeSupplier); - balancer = new BucketBalancer(nodeSupplier, shardManager, true, new Duration(1, DAYS), true, true, "test"); - } - - @AfterEach - public void teardown() - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - } - - @Test - public void testSingleDistributionUnbalanced() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - createAssignments(distributionId, AVAILABLE_WORKERS, 10, 3, 1, 1, 1); - - assertBalancing(balancer, 6); - } - - @Test - public void testSingleDistributionSlightlyUnbalanced() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - createAssignments(distributionId, AVAILABLE_WORKERS, 4, 4, 3, 3, 2); - - assertBalancing(balancer, 1); - } - - @Test - public void testSingleDistributionBalanced() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - createAssignments(distributionId, AVAILABLE_WORKERS, 4, 3, 3, 3, 3); - - assertBalancing(balancer, 0); - } - - @Test - public void testSingleDistributionUnbalancedWithDeadNode() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - ImmutableList nodes = ImmutableList.builder().addAll(AVAILABLE_WORKERS).add("node6").build(); - createAssignments(distributionId, nodes, 11, 1, 1, 1, 1, 1); - - assertBalancing(balancer, 8); - } - - @Test - public void testSingleDistributionUnbalancedWithNewNode() - { - long distributionId = createDistribution("distA", 16); - createBucketedTable("testA", distributionId); - createBucketedTable("testB", distributionId); - - createAssignments(distributionId, AVAILABLE_WORKERS, 12, 1, 1, 1, 1); - nodeManager.addNode(createTestingNode("node6")); - - assertBalancing(balancer, 9); - } - - @Test - public void testMultipleDistributionUnbalanced() - { - long distributionA = createDistribution("distA", 17); - createBucketedTable("testA", distributionA); - createAssignments(distributionA, AVAILABLE_WORKERS, 11, 3, 1, 1, 1); - - long distributionB = createDistribution("distB", 10); - createBucketedTable("testB", distributionB); - createAssignments(distributionB, AVAILABLE_WORKERS, 8, 2, 0, 0, 0); - - long distributionC = createDistribution("distC", 4); - createBucketedTable("testC", distributionC); - createAssignments(distributionC, AVAILABLE_WORKERS, 2, 2, 0, 0, 0); - - assertBalancing(balancer, 15); - } - - @Test - public void testMultipleDistributionUnbalancedWithDiskSpace() - { - long distributionA = createDistribution("distA", 4); - createBucketedTable("testA", distributionA, DataSize.valueOf("4B")); - createAssignments(distributionA, AVAILABLE_WORKERS, 1, 1, 1, 1, 0); - - long distributionB = createDistribution("distB", 4); - createBucketedTable("testB", distributionB, DataSize.valueOf("4B")); - createAssignments(distributionB, AVAILABLE_WORKERS, 1, 1, 1, 0, 1); - - long distributionC = createDistribution("distC", 2); - createBucketedTable("testC", distributionC, DataSize.valueOf("2B")); - createAssignments(distributionC, AVAILABLE_WORKERS, 0, 0, 0, 2, 0); - - assertBalancing(balancer, 1); - - assertThat(balancer.fetchClusterState().getAssignedBytes().values() - .stream() - .distinct() - .count()).isEqualTo(1); - } - - @Test - public void testMultipleDistributionUnbalancedWithDiskSpace2() - { - long distributionA = createDistribution("distA", 4); - createBucketedTable("testA", distributionA, DataSize.valueOf("4B")); - createAssignments(distributionA, AVAILABLE_WORKERS, 1, 1, 1, 1, 0); - - long distributionB = createDistribution("distB", 4); - createBucketedTable("testB", distributionB, DataSize.valueOf("4B")); - createAssignments(distributionB, AVAILABLE_WORKERS, 2, 1, 1, 0, 0); - - assertBalancing(balancer, 1); - } - - @Test - public void testMultipleDistributionUnbalancedWorstCase() - { - // we will end up with only one bucket on node1 - long distributionA = createDistribution("distA", 4); - createBucketedTable("testA", distributionA, DataSize.valueOf("4B")); - createAssignments(distributionA, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - long distributionB = createDistribution("distB", 4); - createBucketedTable("testB", distributionB, DataSize.valueOf("4B")); - createAssignments(distributionB, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - long distributionC = createDistribution("distC", 4); - createBucketedTable("testC", distributionC, DataSize.valueOf("4B")); - createAssignments(distributionC, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - long distributionD = createDistribution("distD", 4); - createBucketedTable("testD", distributionD, DataSize.valueOf("4B")); - createAssignments(distributionD, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - long distributionE = createDistribution("distE", 4); - createBucketedTable("testE", distributionE, DataSize.valueOf("4B")); - createAssignments(distributionE, AVAILABLE_WORKERS, 4, 0, 0, 0, 0); - - assertBalancing(balancer, 15); - } - - private static void assertBalancing(BucketBalancer balancer, int expectedMoves) - { - int actualMoves = balancer.balance(); - assertThat(actualMoves).isEqualTo(expectedMoves); - - // check that number of buckets per node is within bounds - ClusterState clusterState = balancer.fetchClusterState(); - for (Distribution distribution : clusterState.getDistributionAssignments().keySet()) { - Multiset allocationCounts = HashMultiset.create(); - clusterState.getDistributionAssignments().get(distribution).stream() - .map(BucketAssignment::getNodeIdentifier) - .forEach(allocationCounts::add); - - double bucketsPerNode = (1.0 * allocationCounts.size()) / clusterState.getActiveNodes().size(); - for (String node : allocationCounts) { - assertGreaterThanOrEqual(allocationCounts.count(node), (int) Math.floor(bucketsPerNode), node + " has fewer buckets than expected"); - assertLessThanOrEqual(allocationCounts.count(node), (int) Math.ceil(bucketsPerNode), node + " has more buckets than expected"); - } - } - - // check stability - assertThat(balancer.balance()).isEqualTo(0); - } - - private long createDistribution(String distributionName, int bucketCount) - { - MetadataDao dao = dbi.onDemand(MetadataDao.class); - long distributionId = dao.insertDistribution(distributionName, serializeColumnTypes(ImmutableList.of(BIGINT)), bucketCount); - shardManager.createBuckets(distributionId, bucketCount); - return distributionId; - } - - private long createBucketedTable(String tableName, long distributionId) - { - return createBucketedTable(tableName, distributionId, DataSize.valueOf("0B")); - } - - private long createBucketedTable(String tableName, long distributionId, DataSize compressedSize) - { - MetadataDao dao = dbi.onDemand(MetadataDao.class); - long tableId = dao.insertTable("test", tableName, false, false, distributionId, 0); - List columnsA = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columnsA, false, OptionalLong.empty()); - - metadataDao.updateTableStats(tableId, 1024, 1024 * 1024 * 1024, compressedSize.toBytes(), compressedSize.toBytes() * 2); - return tableId; - } - - private List createAssignments(long distributionId, List nodes, int... buckets) - { - checkArgument(nodes.size() == buckets.length); - ImmutableList.Builder assignments = ImmutableList.builder(); - int bucketNumber = 0; - for (int i = 0; i < buckets.length; i++) { - for (int j = 0; j < buckets[i]; j++) { - shardManager.updateBucketAssignment(distributionId, bucketNumber, nodes.get(i)); - assignments.add(bucketNode(bucketNumber, nodes.get(i))); - - bucketNumber++; - } - } - return assignments.build(); - } - - private static BucketNode bucketNode(int bucketNumber, String nodeIdentifier) - { - return new BucketNode(bucketNumber, nodeIdentifier); - } - - private static Node createTestingNode(String nodeIdentifier) - { - return new InternalNode(nodeIdentifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancerConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancerConfig.java deleted file mode 100644 index 6b91e65febf6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestBucketBalancerConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.HOURS; - -public class TestBucketBalancerConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(BucketBalancerConfig.class) - .setBalancerEnabled(true) - .setBalancerInterval(new Duration(6, HOURS))); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("storage.balancer-enabled", "false") - .put("storage.balancer-interval", "5h") - .buildOrThrow(); - - BucketBalancerConfig expected = new BucketBalancerConfig() - .setBalancerEnabled(false) - .setBalancerInterval(new Duration(5, HOURS)); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestFileStorageService.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestFileStorageService.java deleted file mode 100644 index 8fddeefa3818..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestFileStorageService.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableSet; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.storage.FileStorageService.getFileSystemPath; -import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestFileStorageService -{ - private Path temporary; - private FileStorageService store; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - store = new FileStorageService(temporary.toFile()); - store.start(); - } - - @AfterEach - public void tearDown() - throws Exception - { - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testGetFileSystemPath() - { - UUID uuid = UUID.fromString("701e1a79-74f7-4f56-b438-b41e8e7d019d"); - File expected = new File("/test", format("70/1e/%s.orc", uuid)); - assertThat(getFileSystemPath(new File("/test"), uuid)).isEqualTo(expected); - } - - @Test - public void testFilePaths() - { - UUID uuid = UUID.fromString("701e1a79-74f7-4f56-b438-b41e8e7d019d"); - File staging = temporary.resolve("staging").resolve(format("%s.orc", uuid)).toFile(); - File storage = temporary.resolve("storage").resolve("70").resolve("1e").resolve(format("%s.orc", uuid)).toFile(); - File quarantine = temporary.resolve("quarantine").resolve(format("%s.orc", uuid)).toFile(); - assertThat(store.getStagingFile(uuid)).isEqualTo(staging); - assertThat(store.getStorageFile(uuid)).isEqualTo(storage); - assertThat(store.getQuarantineFile(uuid)).isEqualTo(quarantine); - } - - @Test - public void testStop() - throws Exception - { - File staging = temporary.resolve("staging").toFile(); - File storage = temporary.resolve("storage").toFile(); - File quarantine = temporary.resolve("quarantine").toFile(); - - assertThat(staging).isDirectory(); - assertThat(storage).isDirectory(); - assertThat(quarantine).isDirectory(); - - File file = store.getStagingFile(randomUUID()); - store.createParents(file); - assertThat(file).doesNotExist(); - assertThat(file.createNewFile()).isTrue(); - assertThat(file).isFile(); - - store.stop(); - - assertThat(file).doesNotExist(); - assertThat(staging).doesNotExist(); - assertThat(storage).isDirectory(); - assertThat(quarantine).isDirectory(); - } - - @Test - public void testGetStorageShards() - throws Exception - { - Set shards = ImmutableSet.builder() - .add(UUID.fromString("9e7abb51-56b5-4180-9164-ad08ddfe7c63")) - .add(UUID.fromString("bbfc3895-1c3d-4bf4-bca4-7b1198b1759e")) - .build(); - - for (UUID shard : shards) { - File file = store.getStorageFile(shard); - store.createParents(file); - assertThat(file.createNewFile()).isTrue(); - } - - File storage = temporary.resolve("storage").toFile(); - assertThat(new File(storage, "abc").mkdir()).isTrue(); - assertThat(new File(storage, "ab/cd").mkdirs()).isTrue(); - assertThat(new File(storage, format("ab/cd/%s.junk", randomUUID())).createNewFile()).isTrue(); - assertThat(new File(storage, "ab/cd/junk.orc").createNewFile()).isTrue(); - - assertThat(store.getStorageShards()).isEqualTo(shards); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestMissingShardComparator.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestMissingShardComparator.java deleted file mode 100644 index 7c779032c5e9..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestMissingShardComparator.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import org.junit.jupiter.api.Test; - -import static io.trino.plugin.raptor.legacy.storage.ShardRecoveryManager.MissingShardComparator; -import static io.trino.plugin.raptor.legacy.storage.ShardRecoveryManager.MissingShardRunnable; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestMissingShardComparator -{ - @Test - public void testOrdering() - { - MissingShardComparator comparator = new MissingShardComparator(); - assertThat(comparator.compare(new DummyMissingShardRunnable(false), new DummyMissingShardRunnable(false))).isEqualTo(0); - assertThat(comparator.compare(new DummyMissingShardRunnable(false), new DummyMissingShardRunnable(true))).isEqualTo(1); - assertThat(comparator.compare(new DummyMissingShardRunnable(true), new DummyMissingShardRunnable(false))).isEqualTo(-1); - assertThat(comparator.compare(new DummyMissingShardRunnable(true), new DummyMissingShardRunnable(true))).isEqualTo(0); - } - - private static class DummyMissingShardRunnable - implements MissingShardRunnable - { - private final boolean active; - - DummyMissingShardRunnable(boolean active) - { - this.active = active; - } - - @Override - public boolean isActive() - { - return active; - } - - @Override - public void run() - { - throw new UnsupportedOperationException(); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcFileRewriter.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcFileRewriter.java deleted file mode 100644 index 08173185ca61..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcFileRewriter.java +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcRecordReader; -import io.trino.plugin.raptor.legacy.storage.OrcFileRewriter.OrcFileInfo; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.SqlMap; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeSignatureParameter; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.math.BigDecimal; -import java.nio.file.Path; -import java.util.BitSet; -import java.util.List; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.RowPagesBuilder.rowPagesBuilder; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.createReader; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.fileOrcDataSource; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.StructuralTestUtil.arrayBlockOf; -import static io.trino.testing.StructuralTestUtil.arrayBlocksEqual; -import static io.trino.testing.StructuralTestUtil.sqlMapEqual; -import static io.trino.testing.StructuralTestUtil.sqlMapOf; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static java.nio.file.Files.createTempDirectory; -import static java.nio.file.Files.readAllBytes; -import static java.util.UUID.randomUUID; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_CLASS) -@Execution(SAME_THREAD) -public class TestOrcFileRewriter -{ - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - private final Path temporary; - - public TestOrcFileRewriter() - throws IOException - { - temporary = createTempDirectory(null); - } - - @AfterAll - public void tearDown() - throws Exception - { - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testRewrite() - throws Exception - { - ArrayType arrayType = new ArrayType(BIGINT); - ArrayType arrayOfArrayType = new ArrayType(arrayType); - Type mapType = TESTING_TYPE_MANAGER.getParameterizedType(StandardTypes.MAP, ImmutableList.of( - TypeSignatureParameter.typeParameter(createVarcharType(5).getTypeSignature()), - TypeSignatureParameter.typeParameter(BOOLEAN.getTypeSignature()))); - List columnIds = ImmutableList.of(3L, 7L, 9L, 10L, 11L, 12L); - DecimalType decimalType = DecimalType.createDecimalType(4, 4); - - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), arrayType, mapType, arrayOfArrayType, decimalType); - - File file = temporary.resolve(randomUUID().toString()).toFile(); - try (OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello", arrayBlockOf(BIGINT, 1, 2), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), new BigDecimal("2.3")) - .row(777L, "sky", arrayBlockOf(BIGINT, 3, 4), sqlMapOf(createVarcharType(5), BOOLEAN, "k2", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)), new BigDecimal("2.3")) - .row(456L, "bye", arrayBlockOf(BIGINT, 5, 6), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7)), new BigDecimal("2.3")) - .row(888L, "world", arrayBlockOf(BIGINT, 7, 8), sqlMapOf(createVarcharType(5), BOOLEAN, "k4", true), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 8), null), new BigDecimal("2.3")) - .row(999L, "done", arrayBlockOf(BIGINT, 9, 10), sqlMapOf(createVarcharType(5), BOOLEAN, "k5", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 9, 10)), new BigDecimal("2.3")) - .build(); - writer.appendPages(pages); - } - - try (OrcDataSource dataSource = fileOrcDataSource(file)) { - OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); - - assertThat(reader.getReaderRowCount()).isEqualTo(5); - assertThat(reader.getFileRowCount()).isEqualTo(5); - assertThat(reader.getSplitLength()).isEqualTo(file.length()); - - Page page = reader.nextPage(); - assertThat(page.getPositionCount()).isEqualTo(5); - - Block column0 = page.getBlock(0); - assertThat(column0.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column0.isNull(i)).isEqualTo(false); - } - assertThat(BIGINT.getLong(column0, 0)).isEqualTo(123L); - assertThat(BIGINT.getLong(column0, 1)).isEqualTo(777L); - assertThat(BIGINT.getLong(column0, 2)).isEqualTo(456L); - assertThat(BIGINT.getLong(column0, 3)).isEqualTo(888L); - assertThat(BIGINT.getLong(column0, 4)).isEqualTo(999L); - - Block column1 = page.getBlock(1); - assertThat(column1.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column1.isNull(i)).isEqualTo(false); - } - assertThat(createVarcharType(20).getSlice(column1, 0)).isEqualTo(utf8Slice("hello")); - assertThat(createVarcharType(20).getSlice(column1, 1)).isEqualTo(utf8Slice("sky")); - assertThat(createVarcharType(20).getSlice(column1, 2)).isEqualTo(utf8Slice("bye")); - assertThat(createVarcharType(20).getSlice(column1, 3)).isEqualTo(utf8Slice("world")); - assertThat(createVarcharType(20).getSlice(column1, 4)).isEqualTo(utf8Slice("done")); - - Block column2 = page.getBlock(2); - assertThat(column2.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column2.isNull(i)).isEqualTo(false); - } - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 0), arrayBlockOf(BIGINT, 1, 2))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 1), arrayBlockOf(BIGINT, 3, 4))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 2), arrayBlockOf(BIGINT, 5, 6))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 3), arrayBlockOf(BIGINT, 7, 8))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 4), arrayBlockOf(BIGINT, 9, 10))).isTrue(); - - Block column3 = page.getBlock(3); - assertThat(column3.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column3.isNull(i)).isEqualTo(false); - } - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 0), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 1), sqlMapOf(createVarcharType(5), BOOLEAN, "k2", false))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 2), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", true))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 3), sqlMapOf(createVarcharType(5), BOOLEAN, "k4", true))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 4), sqlMapOf(createVarcharType(5), BOOLEAN, "k5", true))).isTrue(); - - Block column4 = page.getBlock(4); - assertThat(column4.getPositionCount()).isEqualTo(5); - for (int i = 0; i < 5; i++) { - assertThat(column4.isNull(i)).isEqualTo(false); - } - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 1), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 3), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 8), null))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 4), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 9, 10)))).isTrue(); - - assertThat(reader.nextPage()).isNull(); - - OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes()); - assertThat(orcFileMetadata).isEqualTo(new OrcFileMetadata(ImmutableMap.builder() - .put(3L, BIGINT.getTypeId()) - .put(7L, createVarcharType(20).getTypeId()) - .put(9L, arrayType.getTypeId()) - .put(10L, mapType.getTypeId()) - .put(11L, arrayOfArrayType.getTypeId()) - .put(12L, decimalType.getTypeId()) - .buildOrThrow())); - } - - BitSet rowsToDelete = new BitSet(5); - rowsToDelete.set(1); - rowsToDelete.set(3); - rowsToDelete.set(4); - - File newFile = temporary.resolve(randomUUID().toString()).toFile(); - OrcFileInfo info = OrcFileRewriter.rewrite(TESTING_TYPE_MANAGER, file, newFile, rowsToDelete); - assertThat(info.getRowCount()).isEqualTo(2); - assertThat(info.getUncompressedSize()).isEqualTo(182); - - try (OrcDataSource dataSource = fileOrcDataSource(newFile)) { - OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); - - assertThat(reader.getReaderRowCount()).isEqualTo(2); - assertThat(reader.getFileRowCount()).isEqualTo(2); - assertThat(reader.getSplitLength()).isEqualTo(newFile.length()); - - Page page = reader.nextPage(); - assertThat(page.getPositionCount()).isEqualTo(2); - - Block column0 = page.getBlock(0); - assertThat(column0.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column0.isNull(i)).isEqualTo(false); - } - assertThat(BIGINT.getLong(column0, 0)).isEqualTo(123L); - assertThat(BIGINT.getLong(column0, 1)).isEqualTo(456L); - - Block column1 = page.getBlock(1); - assertThat(column1.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column1.isNull(i)).isEqualTo(false); - } - assertThat(createVarcharType(20).getSlice(column1, 0)).isEqualTo(utf8Slice("hello")); - assertThat(createVarcharType(20).getSlice(column1, 1)).isEqualTo(utf8Slice("bye")); - - Block column2 = page.getBlock(2); - assertThat(column2.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column2.isNull(i)).isEqualTo(false); - } - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 0), arrayBlockOf(BIGINT, 1, 2))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 1), arrayBlockOf(BIGINT, 5, 6))).isTrue(); - - Block column3 = page.getBlock(3); - assertThat(column3.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column3.isNull(i)).isEqualTo(false); - } - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 0), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true))).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column3, 1), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", true))).isTrue(); - - Block column4 = page.getBlock(4); - assertThat(column4.getPositionCount()).isEqualTo(2); - for (int i = 0; i < 2; i++) { - assertThat(column4.isNull(i)).isEqualTo(false); - } - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 1), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7)))).isTrue(); - - assertThat(reader.nextPage()).isEqualTo(null); - - OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes()); - assertThat(orcFileMetadata).isEqualTo(new OrcFileMetadata(ImmutableMap.builder() - .put(3L, BIGINT.getTypeId()) - .put(7L, createVarcharType(20).getTypeId()) - .put(9L, arrayType.getTypeId()) - .put(10L, mapType.getTypeId()) - .put(11L, arrayOfArrayType.getTypeId()) - .put(12L, decimalType.getTypeId()) - .buildOrThrow())); - } - } - - @Test - public void testRewriteAllRowsDeleted() - throws Exception - { - List columnIds = ImmutableList.of(3L); - List columnTypes = ImmutableList.of(BIGINT); - - File file = temporary.resolve(randomUUID().toString()).toFile(); - try (OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); - } - - BitSet rowsToDelete = new BitSet(); - rowsToDelete.set(0); - rowsToDelete.set(1); - - File newFile = temporary.resolve(randomUUID().toString()).toFile(); - OrcFileInfo info = OrcFileRewriter.rewrite(TESTING_TYPE_MANAGER, file, newFile, rowsToDelete); - assertThat(info.getRowCount()).isEqualTo(0); - assertThat(info.getUncompressedSize()).isEqualTo(0); - - assertThat(newFile).doesNotExist(); - } - - @Test - public void testRewriteNoRowsDeleted() - throws Exception - { - List columnIds = ImmutableList.of(3L); - List columnTypes = ImmutableList.of(BIGINT); - - File file = temporary.resolve(randomUUID().toString()).toFile(); - try (OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); - } - - BitSet rowsToDelete = new BitSet(); - - File newFile = temporary.resolve(randomUUID().toString()).toFile(); - OrcFileInfo info = OrcFileRewriter.rewrite(TESTING_TYPE_MANAGER, file, newFile, rowsToDelete); - assertThat(info.getRowCount()).isEqualTo(2); - assertThat(info.getUncompressedSize()).isEqualTo(18); - - assertThat(readAllBytes(newFile.toPath())).isEqualTo(readAllBytes(file.toPath())); - } - - @Test - public void testUncompressedSize() - throws Exception - { - List columnIds = ImmutableList.of(1L, 2L, 3L, 4L, 5L); - List columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY); - - File file = temporary.resolve(randomUUID().toString()).toFile(); - try (OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - List pages = rowPagesBuilder(columnTypes) - .row(true, 123L, 98.7, "hello", utf8Slice("abc")) - .row(false, 456L, 65.4, "world", utf8Slice("xyz")) - .row(null, null, null, null, null) - .build(); - writer.appendPages(pages); - } - - File newFile = temporary.resolve(randomUUID().toString()).toFile(); - OrcFileInfo info = OrcFileRewriter.rewrite(TESTING_TYPE_MANAGER, file, newFile, new BitSet()); - assertThat(info.getRowCount()).isEqualTo(3); - assertThat(info.getUncompressedSize()).isEqualTo(106); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java deleted file mode 100644 index e1fcc36158f6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java +++ /dev/null @@ -1,692 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import io.airlift.slice.Slice; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcReaderOptions; -import io.trino.orc.OrcRecordReader; -import io.trino.plugin.raptor.legacy.RaptorColumnHandle; -import io.trino.plugin.raptor.legacy.backup.BackupManager; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.backup.FileBackupStore; -import io.trino.plugin.raptor.legacy.metadata.ColumnStats; -import io.trino.plugin.raptor.legacy.metadata.ShardDelta; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; -import io.trino.plugin.raptor.legacy.storage.InMemoryShardRecorder.RecordedShard; -import io.trino.spi.NodeManager; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.predicate.NullableValue; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.SqlDate; -import io.trino.spi.type.SqlTimestamp; -import io.trino.spi.type.SqlVarbinary; -import io.trino.spi.type.Type; -import io.trino.testing.MaterializedResult; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.joda.time.DateTime; -import org.joda.time.Days; -import org.joda.time.chrono.ISOChronology; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.BitSet; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.slice.Slices.wrappedBuffer; -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static io.trino.RowPagesBuilder.rowPagesBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.createReader; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.octets; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.DateTimeTestingUtils.sqlTimestampOf; -import static io.trino.testing.MaterializedResult.materializeSourceDataStream; -import static io.trino.testing.MaterializedResult.resultBuilder; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static java.lang.String.format; -import static java.nio.file.Files.createTempDirectory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Fail.fail; -import static org.joda.time.DateTimeZone.UTC; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestRaptorStorageManager -{ - private static final ISOChronology UTC_CHRONOLOGY = ISOChronology.getInstanceUTC(); - private static final DateTime EPOCH = new DateTime(0, UTC_CHRONOLOGY); - private static final String CURRENT_NODE = "node"; - private static final String CONNECTOR_ID = "test"; - private static final long TRANSACTION_ID = 123; - private static final int DELETION_THREADS = 2; - private static final Duration SHARD_RECOVERY_TIMEOUT = new Duration(30, TimeUnit.SECONDS); - private static final int MAX_SHARD_ROWS = 100; - private static final DataSize MAX_FILE_SIZE = DataSize.of(1, MEGABYTE); - private static final Duration MISSING_SHARD_DISCOVERY = new Duration(5, TimeUnit.MINUTES); - private static final OrcReaderOptions READER_OPTIONS = new OrcReaderOptions() - .withMaxMergeDistance(DataSize.of(1, MEGABYTE)) - .withMaxBufferSize(DataSize.of(1, MEGABYTE)) - .withStreamBufferSize(DataSize.of(1, MEGABYTE)) - .withTinyStripeThreshold(DataSize.of(1, MEGABYTE)); - - private final NodeManager nodeManager = new TestingNodeManager(); - private Handle dummyHandle; - private Path temporary; - private StorageService storageService; - private ShardRecoveryManager recoveryManager; - private FileBackupStore fileBackupStore; - private Optional backupStore; - private InMemoryShardRecorder shardRecorder; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - File directory = temporary.resolve("data").toFile(); - storageService = new FileStorageService(directory); - storageService.start(); - - File backupDirectory = temporary.resolve("backup").toFile(); - fileBackupStore = new FileBackupStore(backupDirectory); - fileBackupStore.start(); - backupStore = Optional.of(fileBackupStore); - - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - - ShardManager shardManager = createShardManager(dbi); - Duration discoveryInterval = new Duration(5, TimeUnit.MINUTES); - recoveryManager = new ShardRecoveryManager(storageService, backupStore, nodeManager, shardManager, discoveryInterval, 10); - - shardRecorder = new InMemoryShardRecorder(); - } - - @AfterEach - public void tearDown() - throws Exception - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testWriter() - throws Exception - { - RaptorStorageManager manager = createRaptorStorageManager(); - - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); - - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - sink.appendPages(pages); - - // shard is not recorded until flush - assertThat(shardRecorder.getShards().size()).isEqualTo(0); - - sink.flush(); - - // shard is recorded after flush - List recordedShards = shardRecorder.getShards(); - assertThat(recordedShards.size()).isEqualTo(1); - - List shards = getFutureValue(sink.commit()); - - assertThat(shards.size()).isEqualTo(1); - ShardInfo shardInfo = Iterables.getOnlyElement(shards); - - UUID shardUuid = shardInfo.getShardUuid(); - File file = storageService.getStorageFile(shardUuid); - File backupFile = fileBackupStore.getBackupFile(shardUuid); - - assertThat(recordedShards.get(0).getTransactionId()).isEqualTo(TRANSACTION_ID); - assertThat(recordedShards.get(0).getShardUuid()).isEqualTo(shardUuid); - - assertThat(shardInfo.getRowCount()).isEqualTo(2); - assertThat(shardInfo.getCompressedSize()).isEqualTo(file.length()); - assertThat(shardInfo.getXxhash64()).isEqualTo(xxhash64(file)); - - // verify primary and backup shard exist - assertThat(file) - .describedAs("primary shard") - .exists(); - assertThat(backupFile) - .describedAs("backup shard") - .exists(); - - assertFileEquals(file, backupFile); - - // remove primary shard to force recovery from backup - assertThat(file.delete()).isTrue(); - assertThat(file.getParentFile().delete()).isTrue(); - assertThat(file).doesNotExist(); - - recoveryManager.restoreFromBackup(shardUuid, shardInfo.getCompressedSize(), OptionalLong.of(shardInfo.getXxhash64())); - - try (OrcDataSource dataSource = manager.openShard(shardUuid, READER_OPTIONS)) { - OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); - - Page page = reader.nextPage(); - assertThat(page.getPositionCount()).isEqualTo(2); - - Block column0 = page.getBlock(0); - assertThat(column0.isNull(0)).isEqualTo(false); - assertThat(column0.isNull(1)).isEqualTo(false); - assertThat(BIGINT.getLong(column0, 0)).isEqualTo(123L); - assertThat(BIGINT.getLong(column0, 1)).isEqualTo(456L); - - Block column1 = page.getBlock(1); - assertThat(createVarcharType(10).getSlice(column1, 0)).isEqualTo(utf8Slice("hello")); - assertThat(createVarcharType(10).getSlice(column1, 1)).isEqualTo(utf8Slice("bye")); - - assertThat(reader.nextPage()).isNull(); - } - } - - @Test - public void testReader() - throws Exception - { - RaptorStorageManager manager = createRaptorStorageManager(); - - List columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE); - - byte[] bytes1 = octets(0x00, 0xFE, 0xFF); - byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80); - - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - - Object[][] doubles = { - {881L, "-inf", null, null, null, Double.NEGATIVE_INFINITY}, - {882L, "+inf", null, null, null, Double.POSITIVE_INFINITY}, - {883L, "nan", null, null, null, Double.NaN}, - {884L, "min", null, null, null, Double.MIN_VALUE}, - {885L, "max", null, null, null, Double.MAX_VALUE}, - {886L, "pzero", null, null, null, 0.0}, - {887L, "nzero", null, null, null, -0.0}, - }; - - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello", wrappedBuffer(bytes1), sqlDate(2001, 8, 22).getDays(), true, 123.45) - .row(null, null, null, null, null, null) - .row(456L, "bye", wrappedBuffer(bytes3), sqlDate(2005, 4, 22).getDays(), false, 987.65) - .rows(doubles) - .build(); - - sink.appendPages(pages); - List shards = getFutureValue(sink.commit()); - - assertThat(shards.size()).isEqualTo(1); - UUID uuid = Iterables.getOnlyElement(shards).getShardUuid(); - - MaterializedResult expected = resultBuilder(SESSION, columnTypes) - .row(123L, "hello", sqlBinary(bytes1), sqlDate(2001, 8, 22), true, 123.45) - .row(null, null, null, null, null, null) - .row(456L, "bye", sqlBinary(bytes3), sqlDate(2005, 4, 22), false, 987.65) - .rows(doubles) - .build(); - - // no tuple domain (all) - TupleDomain tupleDomain = TupleDomain.all(); - - try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) { - MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); - assertThat(result.getRowCount()).isEqualTo(expected.getRowCount()); - assertThat(result).containsExactlyElementsOf(expected); - } - - // tuple domain within the column range - tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(new RaptorColumnHandle("c1", 2, BIGINT), NullableValue.of(BIGINT, 124L))); - - try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) { - MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); - assertThat(result.getRowCount()).isEqualTo(expected.getRowCount()); - } - - // tuple domain outside the column range - tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(new RaptorColumnHandle("c1", 2, BIGINT), NullableValue.of(BIGINT, 122L))); - - try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) { - MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); - assertThat(result.getRowCount()).isEqualTo(0); - } - } - - @Test - public void testRewriter() - throws Exception - { - RaptorStorageManager manager = createRaptorStorageManager(); - - long transactionId = TRANSACTION_ID; - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); - - // create file with 2 rows - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - sink.appendPages(pages); - List shards = getFutureValue(sink.commit()); - - assertThat(shardRecorder.getShards().size()).isEqualTo(1); - - // delete one row - BitSet rowsToDelete = new BitSet(); - rowsToDelete.set(0); - Collection fragments = manager.rewriteShard(transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete); - - Slice shardDelta = Iterables.getOnlyElement(fragments); - ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes()); - ShardInfo shardInfo = Iterables.getOnlyElement(shardDeltas.getNewShards()); - - // check that output file has one row - assertThat(shardInfo.getRowCount()).isEqualTo(1); - - // check that storage file is same as backup file - File storageFile = storageService.getStorageFile(shardInfo.getShardUuid()); - File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid()); - assertFileEquals(storageFile, backupFile); - - // verify recorded shard - List recordedShards = shardRecorder.getShards(); - assertThat(recordedShards.size()).isEqualTo(2); - assertThat(recordedShards.get(1).getTransactionId()).isEqualTo(TRANSACTION_ID); - assertThat(recordedShards.get(1).getShardUuid()).isEqualTo(shardInfo.getShardUuid()); - } - - @Test - public void testWriterRollback() - { - // verify staging directory is empty - File staging = temporary.resolve("data").resolve("staging").toFile(); - assertThat(staging).isDirectory(); - assertThat(staging.list()).isEqualTo(new String[] {}); - - // create a shard in staging - RaptorStorageManager manager = createRaptorStorageManager(); - - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); - - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - sink.appendPages(pages); - - sink.flush(); - - // verify shard exists in staging - String[] files = staging.list(); - assertThat(files).isNotNull(); - String stagingFile = Arrays.stream(files) - .filter(file -> file.endsWith(".orc")) - .findFirst() - .orElseThrow(() -> new AssertionError("file not found in staging")); - - // rollback should cleanup staging files - sink.rollback(); - - files = staging.list(); - assertThat(files).isNotNull(); - assertThat(Arrays.stream(files).noneMatch(stagingFile::equals)).isTrue(); - } - - @Test - public void testShardStatsBigint() - { - List stats = columnStats(types(BIGINT), - row(2L), - row(-3L), - row(5L)); - assertColumnStats(stats, 1, -3L, 5L); - } - - @Test - public void testShardStatsDouble() - { - List stats = columnStats(types(DOUBLE), - row(2.5), - row(-4.1), - row(6.6)); - assertColumnStats(stats, 1, -4.1, 6.6); - } - - @Test - public void testShardStatsBigintDouble() - { - List stats = columnStats(types(BIGINT, DOUBLE), - row(-3L, 6.6), - row(5L, -4.1)); - assertColumnStats(stats, 1, -3L, 5L); - assertColumnStats(stats, 2, -4.1, 6.6); - } - - @Test - public void testShardStatsDoubleMinMax() - { - List stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.MIN_VALUE), - row(4.5)); - assertColumnStats(stats, 1, Double.MIN_VALUE, 4.5); - - stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.MAX_VALUE), - row(4.5)); - assertColumnStats(stats, 1, 3.2, Double.MAX_VALUE); - } - - @Test - public void testShardStatsDoubleNotFinite() - { - List stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.NEGATIVE_INFINITY), - row(4.5)); - assertColumnStats(stats, 1, null, 4.5); - - stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.POSITIVE_INFINITY), - row(4.5)); - assertColumnStats(stats, 1, 3.2, null); - - stats = columnStats(types(DOUBLE), - row(3.2), - row(Double.NaN), - row(4.5)); - assertColumnStats(stats, 1, 3.2, 4.5); - } - - @Test - public void testShardStatsVarchar() - { - List stats = columnStats( - types(createVarcharType(10)), - row(utf8Slice("hello")), - row(utf8Slice("bye")), - row(utf8Slice("foo"))); - assertColumnStats(stats, 1, "bye", "hello"); - } - - @Test - public void testShardStatsBigintVarbinary() - { - List stats = columnStats(types(BIGINT, VARBINARY), - row(5L, wrappedBuffer(octets(0x00))), - row(3L, wrappedBuffer(octets(0x01)))); - assertColumnStats(stats, 1, 3L, 5L); - assertNoColumnStats(stats, 2); - } - - @Test - public void testShardStatsDateTimestamp() - { - long minDate = sqlDate(2001, 8, 22).getDays(); - long maxDate = sqlDate(2005, 4, 22).getDays(); - long maxTimestamp = sqlTimestamp(2002, 4, 13, 6, 7, 8).getMillis(); - long minTimestamp = sqlTimestamp(2001, 3, 15, 9, 10, 11).getMillis(); - - List stats = columnStats(types(DATE, TIMESTAMP_MILLIS), - row(minDate, maxTimestamp), - row(maxDate, minTimestamp)); - assertColumnStats(stats, 1, minDate, maxDate); - assertColumnStats(stats, 2, minTimestamp, maxTimestamp); - } - - @Test - public void testMaxShardRows() - { - RaptorStorageManager manager = createRaptorStorageManager(2, DataSize.of(2, MEGABYTE)); - - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); - - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - sink.appendPages(pages); - assertThat(sink.isFull()).isTrue(); - } - - @Test - public void testMaxFileSize() - { - List columnIds = ImmutableList.of(3L, 7L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(5)); - - List pages = rowPagesBuilder(columnTypes) - .row(123L, "hello") - .row(456L, "bye") - .build(); - - // Set maxFileSize to 1 byte, so adding any page makes the StoragePageSink full - RaptorStorageManager manager = createRaptorStorageManager(20, DataSize.ofBytes(1)); - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - sink.appendPages(pages); - assertThat(sink.isFull()).isTrue(); - } - - private static ConnectorPageSource getPageSource( - RaptorStorageManager manager, - List columnIds, - List columnTypes, - UUID uuid, - TupleDomain tupleDomain) - { - return manager.getPageSource(uuid, OptionalInt.empty(), columnIds, columnTypes, tupleDomain, READER_OPTIONS); - } - - private static StoragePageSink createStoragePageSink(StorageManager manager, List columnIds, List columnTypes) - { - long transactionId = TRANSACTION_ID; - return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); - } - - private RaptorStorageManager createRaptorStorageManager() - { - return createRaptorStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE); - } - - private RaptorStorageManager createRaptorStorageManager(int maxShardRows, DataSize maxFileSize) - { - return createRaptorStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize); - } - - public static RaptorStorageManager createRaptorStorageManager(Jdbi dbi, File temporary) - { - return createRaptorStorageManager(dbi, temporary, MAX_SHARD_ROWS); - } - - public static RaptorStorageManager createRaptorStorageManager(Jdbi dbi, File temporary, int maxShardRows) - { - File directory = new File(temporary, "data"); - StorageService storageService = new FileStorageService(directory); - storageService.start(); - - File backupDirectory = new File(temporary, "backup"); - FileBackupStore fileBackupStore = new FileBackupStore(backupDirectory); - fileBackupStore.start(); - Optional backupStore = Optional.of(fileBackupStore); - - ShardManager shardManager = createShardManager(dbi); - ShardRecoveryManager recoveryManager = new ShardRecoveryManager( - storageService, - backupStore, - new TestingNodeManager(), - shardManager, - MISSING_SHARD_DISCOVERY, - 10); - return createRaptorStorageManager( - storageService, - backupStore, - recoveryManager, - new InMemoryShardRecorder(), - maxShardRows, - MAX_FILE_SIZE); - } - - public static RaptorStorageManager createRaptorStorageManager( - StorageService storageService, - Optional backupStore, - ShardRecoveryManager recoveryManager, - ShardRecorder shardRecorder, - int maxShardRows, - DataSize maxFileSize) - { - return new RaptorStorageManager( - CURRENT_NODE, - storageService, - backupStore, - READER_OPTIONS, - new BackupManager(backupStore, storageService, 1), - recoveryManager, - shardRecorder, - TESTING_TYPE_MANAGER, - CONNECTOR_ID, - DELETION_THREADS, - SHARD_RECOVERY_TIMEOUT, - maxShardRows, - maxFileSize, - DataSize.ofBytes(0)); - } - - private static void assertFileEquals(File actual, File expected) - { - assertThat(actual).hasSameBinaryContentAs(expected); - } - - private static void assertColumnStats(List list, long columnId, Object min, Object max) - { - for (ColumnStats stats : list) { - if (stats.getColumnId() == columnId) { - assertThat(stats.getMin()).isEqualTo(min); - assertThat(stats.getMax()).isEqualTo(max); - return; - } - } - fail(format("no stats for column: %s: %s", columnId, list)); - } - - private static void assertNoColumnStats(List list, long columnId) - { - for (ColumnStats stats : list) { - assertThat(stats.getColumnId()) - .isNotEqualTo(columnId); - } - } - - private static List types(Type... types) - { - return ImmutableList.copyOf(types); - } - - private static Object[] row(Object... values) - { - return values; - } - - private List columnStats(List columnTypes, Object[]... rows) - { - ImmutableList.Builder list = ImmutableList.builder(); - for (long i = 1; i <= columnTypes.size(); i++) { - list.add(i); - } - List columnIds = list.build(); - - RaptorStorageManager manager = createRaptorStorageManager(); - StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); - sink.appendPages(rowPagesBuilder(columnTypes).rows(rows).build()); - List shards = getFutureValue(sink.commit()); - - assertThat(shards.size()).isEqualTo(1); - return Iterables.getOnlyElement(shards).getColumnStats(); - } - - private static SqlVarbinary sqlBinary(byte[] bytes) - { - return new SqlVarbinary(bytes); - } - - private static SqlDate sqlDate(int year, int month, int day) - { - DateTime date = new DateTime(year, month, day, 0, 0, 0, 0, UTC); - return new SqlDate(Days.daysBetween(EPOCH, date).getDays()); - } - - private static SqlTimestamp sqlTimestamp(int year, int month, int day, int hour, int minute, int second) - { - return sqlTimestampOf(3, year, month, day, hour, minute, second, 0); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardEjector.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardEjector.java deleted file mode 100644 index 295d971eb50c..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardEjector.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.units.Duration; -import io.trino.client.NodeVersion; -import io.trino.metadata.InternalNode; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.predicate.TupleDomain; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.RepeatedTest; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.spi.type.BigintType.BIGINT; -import static java.nio.file.Files.createTempDirectory; -import static java.util.UUID.randomUUID; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardEjector -{ - private Jdbi dbi; - private Handle dummyHandle; - private ShardManager shardManager; - private Path dataDir; - private StorageService storageService; - - @BeforeEach - public void setup() - throws IOException - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - shardManager = createShardManager(dbi); - - dataDir = createTempDirectory(null); - storageService = new FileStorageService(dataDir.toFile()); - storageService.start(); - } - - @AfterEach - public void teardown() - throws Exception - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - if (dataDir != null) { - deleteRecursively(dataDir, ALLOW_INSECURE); - } - } - - @RepeatedTest(20) - public void testEjector() - throws Exception - { - NodeManager nodeManager = createNodeManager("node1", "node2", "node3", "node4", "node5"); - - ShardEjector ejector = new ShardEjector( - nodeManager.getCurrentNode().getNodeIdentifier(), - nodeManager::getWorkerNodes, - shardManager, - storageService, - new Duration(1, HOURS), - Optional.of(new TestingBackupStore()), - "test"); - - List shards = ImmutableList.builder() - .add(shardInfo("node1", 14)) - .add(shardInfo("node1", 13)) - .add(shardInfo("node1", 12)) - .add(shardInfo("node1", 11)) - .add(shardInfo("node1", 10)) - .add(shardInfo("node1", 10)) - .add(shardInfo("node1", 10)) - .add(shardInfo("node1", 10)) - .add(shardInfo("node2", 5)) - .add(shardInfo("node2", 5)) - .add(shardInfo("node3", 10)) - .add(shardInfo("node4", 10)) - .add(shardInfo("node5", 10)) - .add(shardInfo("node6", 200)) - .build(); - - long tableId = createTable("test"); - List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); - - for (ShardInfo shard : shards.subList(0, 8)) { - File file = storageService.getStorageFile(shard.getShardUuid()); - storageService.createParents(file); - assertThat(file.createNewFile()).isTrue(); - } - - ejector.process(); - - shardManager.getShardNodes(tableId, TupleDomain.all()); - - Set ejectedShards = shards.subList(0, 4).stream() - .map(ShardInfo::getShardUuid) - .collect(toSet()); - Set keptShards = shards.subList(4, 8).stream() - .map(ShardInfo::getShardUuid) - .collect(toSet()); - - Set remaining = uuids(shardManager.getNodeShards("node1")); - - for (UUID uuid : ejectedShards) { - assertThat(remaining).doesNotContain(uuid); - assertThat(storageService.getStorageFile(uuid)).doesNotExist(); - } - - assertThat(remaining).isEqualTo(keptShards); - for (UUID uuid : keptShards) { - assertThat(storageService.getStorageFile(uuid)).exists(); - } - - Set others = ImmutableSet.builder() - .addAll(uuids(shardManager.getNodeShards("node2"))) - .addAll(uuids(shardManager.getNodeShards("node3"))) - .addAll(uuids(shardManager.getNodeShards("node4"))) - .addAll(uuids(shardManager.getNodeShards("node5"))) - .build(); - - assertThat(others).containsAll(ejectedShards); - } - - private long createTable(String name) - { - return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0); - } - - private static Set uuids(Set metadata) - { - return metadata.stream() - .map(ShardMetadata::getShardUuid) - .collect(toSet()); - } - - private static ShardInfo shardInfo(String node, long size) - { - return new ShardInfo(randomUUID(), OptionalInt.empty(), ImmutableSet.of(node), ImmutableList.of(), 1, size, size * 2, 0); - } - - private static NodeManager createNodeManager(String current, String... others) - { - Node currentNode = createTestingNode(current); - TestingNodeManager nodeManager = new TestingNodeManager(currentNode); - for (String other : others) { - nodeManager.addNode(createTestingNode(other)); - } - return nodeManager; - } - - private static Node createTestingNode(String identifier) - { - return new InternalNode(identifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); - } - - private static class TestingBackupStore - implements BackupStore - { - @Override - public void backupShard(UUID uuid, File source) - { - throw new UnsupportedOperationException(); - } - - @Override - public void restoreShard(UUID uuid, File target) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean deleteShard(UUID uuid) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean shardExists(UUID uuid) - { - return true; - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java deleted file mode 100644 index 9199ab498c75..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.io.Files; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.backup.BackupStore; -import io.trino.plugin.raptor.legacy.backup.FileBackupStore; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.spi.TrinoException; -import io.trino.testing.TestingNodeManager; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.UUID; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; -import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; -import static java.nio.file.Files.createTempDirectory; -import static java.nio.file.Files.createTempFile; -import static java.nio.file.Files.writeString; -import static java.util.concurrent.TimeUnit.MINUTES; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardRecovery -{ - private StorageService storageService; - private ShardRecoveryManager recoveryManager; - private Handle dummyHandle; - private Path temporary; - private FileBackupStore backupStore; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - File directory = temporary.resolve("data").toFile(); - File backupDirectory = temporary.resolve("backup").toFile(); - backupStore = new FileBackupStore(backupDirectory); - backupStore.start(); - storageService = new FileStorageService(directory); - storageService.start(); - - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - ShardManager shardManager = createShardManager(dbi); - recoveryManager = createShardRecoveryManager(storageService, Optional.of(backupStore), shardManager); - } - - @AfterEach - public void tearDown() - throws Exception - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testShardRecovery() - throws Exception - { - UUID shardUuid = UUID.randomUUID(); - File file = storageService.getStorageFile(shardUuid); - File tempFile = createTempFile(temporary, "tmp", null).toFile(); - - writeString(tempFile.toPath(), "test data"); - - backupStore.backupShard(shardUuid, tempFile); - assertThat(backupStore.shardExists(shardUuid)).isTrue(); - File backupFile = backupStore.getBackupFile(shardUuid); - assertThat(backupFile).exists(); - assertThat(backupFile.length()).isEqualTo(tempFile.length()); - - assertThat(file).doesNotExist(); - recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.empty()); - assertThat(file).exists(); - assertThat(file.length()).isEqualTo(tempFile.length()); - } - - @Test - public void testShardRecoveryExistingFileSizeMismatch() - throws Exception - { - UUID shardUuid = UUID.randomUUID(); - - // write data and backup - File tempFile = createTempFile(temporary, "tmp", null).toFile(); - writeString(tempFile.toPath(), "test data"); - - backupStore.backupShard(shardUuid, tempFile); - assertThat(backupStore.shardExists(shardUuid)).isTrue(); - - File backupFile = backupStore.getBackupFile(shardUuid); - assertThat(Files.equal(tempFile, backupFile)).isTrue(); - - // write corrupt storage file with wrong length - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); - - writeString(storageFile.toPath(), "bad data"); - - assertThat(storageFile).exists(); - assertThat(storageFile.length()) - .isNotEqualTo(tempFile.length()); - assertThat(Files.equal(storageFile, tempFile)).isFalse(); - - // restore from backup and verify - recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.empty()); - - assertThat(storageFile).exists(); - assertThat(Files.equal(storageFile, tempFile)).isTrue(); - - // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); - assertThat(quarantined.size()).isEqualTo(1); - assertThat(getOnlyElement(quarantined)).startsWith(shardUuid + ".orc.corrupt"); - } - - @Test - public void testShardRecoveryExistingFileChecksumMismatch() - throws Exception - { - UUID shardUuid = UUID.randomUUID(); - - // write data and backup - File tempFile = createTempFile(temporary, "tmp", null).toFile(); - writeString(tempFile.toPath(), "test data"); - - backupStore.backupShard(shardUuid, tempFile); - assertThat(backupStore.shardExists(shardUuid)).isTrue(); - - File backupFile = backupStore.getBackupFile(shardUuid); - assertThat(Files.equal(tempFile, backupFile)).isTrue(); - - // write corrupt storage file with wrong data - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); - - writeString(storageFile.toPath(), "test xata"); - - assertThat(storageFile).exists(); - assertThat(storageFile.length()).isEqualTo(tempFile.length()); - assertThat(Files.equal(storageFile, tempFile)).isFalse(); - - // restore from backup and verify - recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.of(xxhash64(tempFile))); - - assertThat(storageFile).exists(); - assertThat(Files.equal(storageFile, tempFile)).isTrue(); - - // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); - assertThat(quarantined.size()).isEqualTo(1); - assertThat(getOnlyElement(quarantined)).startsWith(shardUuid + ".orc.corrupt"); - } - - @Test - public void testShardRecoveryBackupChecksumMismatch() - throws Exception - { - UUID shardUuid = UUID.randomUUID(); - - // write storage file - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); - - writeString(storageFile.toPath(), "test data"); - - long size = storageFile.length(); - long xxhash64 = xxhash64(storageFile); - - // backup and verify - backupStore.backupShard(shardUuid, storageFile); - - assertThat(backupStore.shardExists(shardUuid)).isTrue(); - File backupFile = backupStore.getBackupFile(shardUuid); - assertThat(Files.equal(storageFile, backupFile)).isTrue(); - - // corrupt backup file - writeString(backupFile.toPath(), "test xata"); - - assertThat(backupFile).exists(); - assertThat(storageFile.length()).isEqualTo(backupFile.length()); - assertThat(Files.equal(storageFile, backupFile)).isFalse(); - - // delete local file to force restore - assertThat(storageFile.delete()).isTrue(); - assertThat(storageFile).doesNotExist(); - - // restore should fail - assertTrinoExceptionThrownBy(() -> recoveryManager.restoreFromBackup(shardUuid, size, OptionalLong.of(xxhash64))) - .hasErrorCode(RAPTOR_BACKUP_CORRUPTION) - .hasMessage("Backup is corrupt after read: %s", shardUuid); - - // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); - assertThat(quarantined.size()).isEqualTo(1); - assertThat(getOnlyElement(quarantined)).startsWith(shardUuid + ".orc.corrupt"); - } - - @Test - public void testNoBackupException() - { - assertThatThrownBy(() -> { - recoveryManager.restoreFromBackup(UUID.randomUUID(), 0, OptionalLong.empty()); - }) - .isInstanceOf(TrinoException.class) - .hasMessageMatching("No backup file found for shard: .*"); - } - - public static ShardRecoveryManager createShardRecoveryManager( - StorageService storageService, - Optional backupStore, - ShardManager shardManager) - { - return new ShardRecoveryManager( - storageService, - backupStore, - new TestingNodeManager(), - shardManager, - new Duration(5, MINUTES), - 10); - } - - private static List listFiles(File path) - { - String[] files = path.list(); - assertThat(files).isNotNull(); - return ImmutableList.copyOf(files); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardWriter.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardWriter.java deleted file mode 100644 index 7eb3d6e53ed5..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardWriter.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.trino.RowPagesBuilder; -import io.trino.orc.OrcDataSource; -import io.trino.orc.OrcRecordReader; -import io.trino.spi.Page; -import io.trino.spi.block.Block; -import io.trino.spi.block.SqlMap; -import io.trino.spi.classloader.ThreadContextClassLoader; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeSignatureParameter; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.slice.Slices.wrappedBuffer; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.createReader; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.fileOrcDataSource; -import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.octets; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.StructuralTestUtil.arrayBlockOf; -import static io.trino.testing.StructuralTestUtil.arrayBlocksEqual; -import static io.trino.testing.StructuralTestUtil.sqlMapEqual; -import static io.trino.testing.StructuralTestUtil.sqlMapOf; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static java.nio.file.Files.createTempDirectory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; - -@TestInstance(PER_CLASS) -@Execution(CONCURRENT) -public class TestShardWriter -{ - private final Path directory; - - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - public TestShardWriter() - throws IOException - { - directory = createTempDirectory(null); - } - - @AfterAll - public void tearDown() - throws Exception - { - deleteRecursively(directory, ALLOW_INSECURE); - } - - @Test - public void testWriter() - throws Exception - { - List columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L); - ArrayType arrayType = new ArrayType(BIGINT); - ArrayType arrayOfArrayType = new ArrayType(arrayType); - Type mapType = TESTING_TYPE_MANAGER.getParameterizedType(StandardTypes.MAP, ImmutableList.of( - TypeSignatureParameter.typeParameter(createVarcharType(10).getTypeSignature()), - TypeSignatureParameter.typeParameter(BOOLEAN.getTypeSignature()))); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType); - File file = directory.resolve(System.nanoTime() + ".orc").toFile(); - - byte[] bytes1 = octets(0x00, 0xFE, 0xFF); - byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80); - - RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(columnTypes) - .row(123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))) - .row(null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), sqlMapOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))) - .row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT))); - - try (ThreadContextClassLoader _ = new ThreadContextClassLoader(new EmptyClassLoader()); - OrcFileWriter writer = new OrcFileWriter(TESTING_TYPE_MANAGER, columnIds, columnTypes, file)) { - writer.appendPages(rowPagesBuilder.build()); - } - - try (OrcDataSource dataSource = fileOrcDataSource(file)) { - OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); - assertThat(reader.getReaderRowCount()).isEqualTo(3); - assertThat(reader.getReaderPosition()).isEqualTo(0); - assertThat(reader.getFileRowCount()).isEqualTo(reader.getReaderRowCount()); - assertThat(reader.getFilePosition()).isEqualTo(reader.getFilePosition()); - - Page page = reader.nextPage(); - assertThat(page.getPositionCount()).isEqualTo(3); - assertThat(reader.getReaderPosition()).isEqualTo(0); - assertThat(reader.getFilePosition()).isEqualTo(reader.getFilePosition()); - - Block column0 = page.getBlock(0); - assertThat(column0.isNull(0)).isEqualTo(false); - assertThat(column0.isNull(1)).isEqualTo(true); - assertThat(column0.isNull(2)).isEqualTo(false); - assertThat(BIGINT.getLong(column0, 0)).isEqualTo(123L); - assertThat(BIGINT.getLong(column0, 2)).isEqualTo(456L); - - Block column1 = page.getBlock(1); - assertThat(createVarcharType(10).getSlice(column1, 0)).isEqualTo(utf8Slice("hello")); - assertThat(createVarcharType(10).getSlice(column1, 1)).isEqualTo(utf8Slice("world")); - assertThat(createVarcharType(10).getSlice(column1, 2)).isEqualTo(utf8Slice("bye \u2603")); - - Block column2 = page.getBlock(2); - assertThat(VARBINARY.getSlice(column2, 0)).isEqualTo(wrappedBuffer(bytes1)); - assertThat(column2.isNull(1)).isEqualTo(true); - assertThat(VARBINARY.getSlice(column2, 2)).isEqualTo(wrappedBuffer(bytes3)); - - Block column3 = page.getBlock(3); - assertThat(column3.isNull(0)).isEqualTo(false); - assertThat(column3.isNull(1)).isEqualTo(false); - assertThat(column3.isNull(2)).isEqualTo(false); - assertThat(DOUBLE.getDouble(column3, 0)).isEqualTo(123.456); - assertThat(DOUBLE.getDouble(column3, 1)).isEqualTo(Double.POSITIVE_INFINITY); - assertThat(DOUBLE.getDouble(column3, 2)).isNaN(); - - Block column4 = page.getBlock(4); - assertThat(column4.isNull(0)).isEqualTo(false); - assertThat(column4.isNull(1)).isEqualTo(true); - assertThat(column4.isNull(2)).isEqualTo(false); - assertThat(BOOLEAN.getBoolean(column4, 0)).isEqualTo(true); - assertThat(BOOLEAN.getBoolean(column4, 2)).isEqualTo(false); - - Block column5 = page.getBlock(5); - assertThat(column5.getPositionCount()).isEqualTo(3); - - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 0), arrayBlockOf(BIGINT, 1, 2))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 1), arrayBlockOf(BIGINT, 3, null))).isTrue(); - assertThat(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 2), arrayBlockOf(BIGINT))).isTrue(); - - Block column6 = page.getBlock(6); - assertThat(column6.getPositionCount()).isEqualTo(3); - - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column6, 0), sqlMapOf(createVarcharType(5), BOOLEAN, "k1", true))).isTrue(); - SqlMap object = (SqlMap) mapType.getObject(column6, 1); - SqlMap k2 = sqlMapOf(createVarcharType(5), BOOLEAN, "k2", null); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, object, k2)).isTrue(); - assertThat(sqlMapEqual(createVarcharType(5), BOOLEAN, (SqlMap) mapType.getObject(column6, 2), sqlMapOf(createVarcharType(5), BOOLEAN, "k3", false))).isTrue(); - - Block column7 = page.getBlock(7); - assertThat(column7.getPositionCount()).isEqualTo(3); - - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 1), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)))).isTrue(); - assertThat(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)))).isTrue(); - - assertThat(reader.nextPage()).isNull(); - assertThat(reader.getReaderPosition()).isEqualTo(3); - assertThat(reader.getFilePosition()).isEqualTo(reader.getFilePosition()); - - OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes()); - assertThat(orcFileMetadata).isEqualTo(new OrcFileMetadata(ImmutableMap.builder() - .put(1L, BIGINT.getTypeId()) - .put(2L, createVarcharType(10).getTypeId()) - .put(4L, VARBINARY.getTypeId()) - .put(6L, DOUBLE.getTypeId()) - .put(7L, BOOLEAN.getTypeId()) - .put(8L, arrayType.getTypeId()) - .put(9L, mapType.getTypeId()) - .put(10L, arrayOfArrayType.getTypeId()) - .buildOrThrow())); - } - - File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc"); - assertThat(crcFile).doesNotExist(); - } - - @SuppressWarnings("EmptyClass") - private static class EmptyClassLoader - extends ClassLoader - { - protected EmptyClassLoader() - { - super(null); - } - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestStorageManagerConfig.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestStorageManagerConfig.java deleted file mode 100644 index 17deeaeb059e..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestStorageManagerConfig.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.DataSize; -import io.airlift.units.Duration; -import jakarta.validation.constraints.NotNull; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static io.airlift.testing.ValidationAssertions.assertFailsValidation; -import static io.airlift.units.DataSize.Unit.GIGABYTE; -import static io.airlift.units.DataSize.Unit.KILOBYTE; -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static java.lang.Math.max; -import static java.lang.Runtime.getRuntime; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; - -public class TestStorageManagerConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(StorageManagerConfig.class) - .setDataDirectory(null) - .setMinAvailableSpace(DataSize.ofBytes(0)) - .setOrcMaxMergeDistance(DataSize.of(1, MEGABYTE)) - .setOrcMaxReadSize(DataSize.of(8, MEGABYTE)) - .setOrcStreamBufferSize(DataSize.of(8, MEGABYTE)) - .setOrcTinyStripeThreshold(DataSize.of(8, MEGABYTE)) - .setOrcLazyReadSmallRanges(true) - .setOrcNestedLazy(true) - .setDeletionThreads(max(1, getRuntime().availableProcessors() / 2)) - .setShardRecoveryTimeout(new Duration(30, SECONDS)) - .setMissingShardDiscoveryInterval(new Duration(5, MINUTES)) - .setCompactionInterval(new Duration(1, HOURS)) - .setShardEjectorInterval(new Duration(4, HOURS)) - .setRecoveryThreads(10) - .setOrganizationThreads(5) - .setCompactionEnabled(true) - .setOrganizationEnabled(true) - .setOrganizationInterval(new Duration(7, DAYS)) - .setOrganizationDiscoveryInterval(new Duration(6, HOURS)) - .setMaxShardRows(1_000_000) - .setMaxShardSize(DataSize.of(256, MEGABYTE)) - .setMaxBufferSize(DataSize.of(256, MEGABYTE)) - .setOneSplitPerBucketThreshold(0)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("storage.data-directory", "/data") - .put("storage.min-available-space", "123GB") - .put("storage.orc.max-merge-distance", "16kB") - .put("storage.orc.max-read-size", "16kB") - .put("storage.orc.stream-buffer-size", "16kB") - .put("storage.orc.tiny-stripe-threshold", "15kB") - .put("storage.orc.lazy-read-small-ranges", "false") - .put("storage.orc.nested-lazy", "false") - .put("storage.max-deletion-threads", "999") - .put("storage.shard-recovery-timeout", "1m") - .put("storage.missing-shard-discovery-interval", "4m") - .put("storage.compaction-enabled", "false") - .put("storage.compaction-interval", "4h") - .put("storage.organization-enabled", "false") - .put("storage.organization-interval", "4h") - .put("storage.organization-discovery-interval", "2h") - .put("storage.ejector-interval", "9h") - .put("storage.max-recovery-threads", "12") - .put("storage.max-organization-threads", "12") - .put("storage.max-shard-rows", "10000") - .put("storage.max-shard-size", "10MB") - .put("storage.max-buffer-size", "512MB") - .put("storage.one-split-per-bucket-threshold", "4") - .buildOrThrow(); - - StorageManagerConfig expected = new StorageManagerConfig() - .setDataDirectory(new File("/data")) - .setMinAvailableSpace(DataSize.of(123, GIGABYTE)) - .setOrcMaxMergeDistance(DataSize.of(16, KILOBYTE)) - .setOrcMaxReadSize(DataSize.of(16, KILOBYTE)) - .setOrcStreamBufferSize(DataSize.of(16, KILOBYTE)) - .setOrcTinyStripeThreshold(DataSize.of(15, KILOBYTE)) - .setOrcLazyReadSmallRanges(false) - .setOrcNestedLazy(false) - .setDeletionThreads(999) - .setShardRecoveryTimeout(new Duration(1, MINUTES)) - .setMissingShardDiscoveryInterval(new Duration(4, MINUTES)) - .setCompactionEnabled(false) - .setCompactionInterval(new Duration(4, HOURS)) - .setOrganizationEnabled(false) - .setOrganizationInterval(new Duration(4, HOURS)) - .setOrganizationDiscoveryInterval(new Duration(2, HOURS)) - .setShardEjectorInterval(new Duration(9, HOURS)) - .setRecoveryThreads(12) - .setOrganizationThreads(12) - .setMaxShardRows(10_000) - .setMaxShardSize(DataSize.of(10, MEGABYTE)) - .setMaxBufferSize(DataSize.of(512, MEGABYTE)) - .setOneSplitPerBucketThreshold(4); - - assertFullMapping(properties, expected); - } - - @Test - public void testValidations() - { - assertFailsValidation(new StorageManagerConfig().setDataDirectory(null), "dataDirectory", "must not be null", NotNull.class); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestCompactionSetCreator.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestCompactionSetCreator.java deleted file mode 100644 index 149a0cfbaed4..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestCompactionSetCreator.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.units.DataSize; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import java.time.Duration; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestCompactionSetCreator -{ - private static final long MAX_SHARD_ROWS = 100; - private static final DataSize MAX_SHARD_SIZE = DataSize.ofBytes(100); - private static final Table tableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.empty(), false); - private static final Table temporalTableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.of(1), false); - private static final Table bucketedTableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.of(3), OptionalLong.empty(), false); - private static final Table bucketedTemporalTableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.of(3), OptionalLong.of(1), false); - - private final CompactionSetCreator compactionSetCreator = new CompactionSetCreator(MAX_SHARD_SIZE, MAX_SHARD_ROWS); - - @Test - public void testNonTemporalOrganizationSetSimple() - { - List inputShards = ImmutableList.of( - shardWithSize(10, 10), - shardWithSize(10, 10), - shardWithSize(10, 10)); - - Set compactionSets = compactionSetCreator.createCompactionSets(tableInfo, inputShards); - assertThat(compactionSets.size()).isEqualTo(1); - assertThat(getOnlyElement(compactionSets).getShards()).isEqualTo(extractIndexes(inputShards, 0, 1, 2)); - } - - @Test - public void testNonTemporalSizeBasedOrganizationSet() - { - List inputShards = ImmutableList.of( - shardWithSize(10, 70), - shardWithSize(10, 20), - shardWithSize(10, 30), - shardWithSize(10, 120)); - - Set compactionSets = compactionSetCreator.createCompactionSets(tableInfo, inputShards); - - Set actual = new HashSet<>(); - for (OrganizationSet set : compactionSets) { - actual.addAll(set.getShards()); - } - assertThat(extractIndexes(inputShards, 0, 1, 2)).containsAll(actual); - } - - @Test - public void testNonTemporalRowCountBasedOrganizationSet() - { - List inputShards = ImmutableList.of( - shardWithSize(50, 10), - shardWithSize(100, 10), - shardWithSize(20, 10), - shardWithSize(30, 10)); - - Set compactionSets = compactionSetCreator.createCompactionSets(tableInfo, inputShards); - - Set actual = new HashSet<>(); - for (OrganizationSet set : compactionSets) { - actual.addAll(set.getShards()); - } - - assertThat(extractIndexes(inputShards, 0, 2, 3)).containsAll(actual); - } - - @Test - public void testTemporalCompactionNoCompactionAcrossDays() - { - long day1 = Duration.ofDays(Duration.ofNanos(System.nanoTime()).toDays()).toMillis(); - long day2 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 1).toMillis(); - long day3 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 2).toMillis(); - - List inputShards = ImmutableList.of( - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day1), - shardWithTemporalRange(TIMESTAMP_MILLIS, day2, day2), - shardWithTemporalRange(TIMESTAMP_MILLIS, day2, day2), - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day1), - shardWithTemporalRange(TIMESTAMP_MILLIS, day3, day3)); - - Set actual = compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards); - assertThat(actual.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(temporalTableInfo.getTableId(), extractIndexes(inputShards, 0, 3), OptionalInt.empty()), - new OrganizationSet(temporalTableInfo.getTableId(), extractIndexes(inputShards, 1, 2), OptionalInt.empty())); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testTemporalCompactionSpanningDays() - { - long day1 = Duration.ofDays(Duration.ofNanos(System.nanoTime()).toDays()).toMillis(); - long day2 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 1).toMillis(); - long day3 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 2).toMillis(); - long day4 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 3).toMillis(); - - List inputShards = ImmutableList.of( - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day3), // day2 - shardWithTemporalRange(TIMESTAMP_MILLIS, day2, day2), // day2 - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day1), // day1 - shardWithTemporalRange(TIMESTAMP_MILLIS, day1 + 100, day2 + 100), // day1 - shardWithTemporalRange(TIMESTAMP_MILLIS, day1 - 100, day2 - 100), // day1 - shardWithTemporalRange(TIMESTAMP_MILLIS, day2 - 100, day3 - 100), // day2 - shardWithTemporalRange(TIMESTAMP_MILLIS, day1, day4)); // day2 - - long tableId = temporalTableInfo.getTableId(); - Set compactionSets = compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards); - - assertThat(compactionSets.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(tableId, extractIndexes(inputShards, 0, 1, 5, 6), OptionalInt.empty()), - new OrganizationSet(tableId, extractIndexes(inputShards, 2, 3, 4), OptionalInt.empty())); - assertThat(compactionSets).isEqualTo(expected); - } - - @Test - public void testTemporalCompactionDate() - { - long day1 = Duration.ofNanos(System.nanoTime()).toDays(); - long day2 = day1 + 1; - long day3 = day1 + 2; - - List inputShards = ImmutableList.of( - shardWithTemporalRange(DATE, day1, day1), - shardWithTemporalRange(DATE, day2, day2), - shardWithTemporalRange(DATE, day3, day3), - shardWithTemporalRange(DATE, day1, day3), - shardWithTemporalRange(DATE, day2, day3), - shardWithTemporalRange(DATE, day1, day2)); - - long tableId = temporalTableInfo.getTableId(); - Set actual = compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards); - - assertThat(actual.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(tableId, extractIndexes(inputShards, 0, 3, 5), OptionalInt.empty()), - new OrganizationSet(tableId, extractIndexes(inputShards, 1, 4), OptionalInt.empty())); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testBucketedTableCompaction() - { - List inputShards = ImmutableList.of( - shardWithBucket(1), - shardWithBucket(2), - shardWithBucket(2), - shardWithBucket(1), - shardWithBucket(2), - shardWithBucket(1)); - - long tableId = bucketedTableInfo.getTableId(); - Set actual = compactionSetCreator.createCompactionSets(bucketedTableInfo, inputShards); - - assertThat(actual.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(tableId, extractIndexes(inputShards, 0, 3, 5), OptionalInt.of(1)), - new OrganizationSet(tableId, extractIndexes(inputShards, 1, 2, 4), OptionalInt.of(2))); - assertThat(actual).isEqualTo(expected); - } - - static Set extractIndexes(List inputShards, int... indexes) - { - ImmutableSet.Builder builder = ImmutableSet.builder(); - for (int index : indexes) { - builder.add(inputShards.get(index).getShardUuid()); - } - return builder.build(); - } - - @Test - public void testBucketedTemporalTableCompaction() - { - long day1 = 1; - long day2 = 2; - long day3 = 3; - long day4 = 4; - - List inputShards = ImmutableList.of( - shardWithTemporalBucket(OptionalInt.of(1), DATE, day1, day1), - shardWithTemporalBucket(OptionalInt.of(2), DATE, day2, day2), - shardWithTemporalBucket(OptionalInt.of(1), DATE, day1, day1), - shardWithTemporalBucket(OptionalInt.of(2), DATE, day2, day2), - shardWithTemporalBucket(OptionalInt.of(1), DATE, day3, day3), - shardWithTemporalBucket(OptionalInt.of(2), DATE, day4, day4)); - - long tableId = bucketedTemporalTableInfo.getTableId(); - Set actual = compactionSetCreator.createCompactionSets(bucketedTemporalTableInfo, inputShards); - - assertThat(actual.size()).isEqualTo(2); - - Set expected = ImmutableSet.of( - new OrganizationSet(tableId, extractIndexes(inputShards, 0, 2), OptionalInt.of(1)), - new OrganizationSet(tableId, extractIndexes(inputShards, 1, 3), OptionalInt.of(2))); - assertThat(actual).isEqualTo(expected); - } - - private static ShardIndexInfo shardWithSize(long rows, long size) - { - return new ShardIndexInfo( - 1, - OptionalInt.empty(), - UUID.randomUUID(), - rows, - size, - Optional.empty(), - Optional.empty()); - } - - private static ShardIndexInfo shardWithTemporalRange(Type type, Long start, Long end) - { - return shardWithTemporalBucket(OptionalInt.empty(), type, start, end); - } - - private static ShardIndexInfo shardWithBucket(int bucketNumber) - { - return new ShardIndexInfo( - 1, - OptionalInt.of(bucketNumber), - UUID.randomUUID(), - 1, - 1, - Optional.empty(), - Optional.empty()); - } - - private static ShardIndexInfo shardWithTemporalBucket(OptionalInt bucketNumber, Type type, Long start, Long end) - { - if (type.equals(DATE)) { - return new ShardIndexInfo( - 1, - bucketNumber, - UUID.randomUUID(), - 1, - 1, - Optional.empty(), - Optional.of(ShardRange.of(new Tuple(type, start.intValue()), new Tuple(type, end.intValue())))); - } - return new ShardIndexInfo( - 1, - bucketNumber, - UUID.randomUUID(), - 1, - 1, - Optional.empty(), - Optional.of(ShardRange.of(new Tuple(type, start), new Tuple(type, end)))); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java deleted file mode 100644 index daf49460b080..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import io.airlift.units.DataSize; -import io.trino.SequencePageBuilder; -import io.trino.operator.PagesIndex; -import io.trino.operator.PagesIndexPageSorter; -import io.trino.orc.OrcReaderOptions; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.RaptorStorageManager; -import io.trino.plugin.raptor.legacy.storage.StorageManager; -import io.trino.plugin.raptor.legacy.storage.StoragePageSink; -import io.trino.spi.Page; -import io.trino.spi.PageBuilder; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ConnectorPageSource; -import io.trino.spi.connector.SortOrder; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeOperators; -import io.trino.testing.MaterializedResult; -import io.trino.testing.MaterializedRow; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.storage.TestRaptorStorageManager.createRaptorStorageManager; -import static io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.MaterializedResult.materializeSourceDataStream; -import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static java.nio.file.Files.createTempDirectory; -import static java.util.Collections.nCopies; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardCompactor -{ - private static final int MAX_SHARD_ROWS = 1000; - private static final PagesIndexPageSorter PAGE_SORTER = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); - private static final OrcReaderOptions READER_OPTIONS = new OrcReaderOptions() - .withMaxMergeDistance(DataSize.of(1, MEGABYTE)) - .withMaxBufferSize(DataSize.of(1, MEGABYTE)) - .withStreamBufferSize(DataSize.of(1, MEGABYTE)) - .withTinyStripeThreshold(DataSize.of(1, MEGABYTE)); - - private RaptorStorageManager storageManager; - private ShardCompactor compactor; - private Path temporary; - private Handle dummyHandle; - - @BeforeEach - public void setup() - throws IOException - { - temporary = createTempDirectory(null); - Jdbi dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - storageManager = createRaptorStorageManager(dbi, temporary.toFile(), MAX_SHARD_ROWS); - compactor = new ShardCompactor(storageManager, READER_OPTIONS, new TypeOperators()); - } - - @AfterEach - public void tearDown() - throws Exception - { - if (dummyHandle != null) { - dummyHandle.close(); - dummyHandle = null; - } - deleteRecursively(temporary, ALLOW_INSECURE); - } - - @Test - public void testShardCompactor() - throws Exception - { - List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP_MILLIS); - - List inputShards = createShards(storageManager, columnIds, columnTypes, 3); - assertThat(inputShards.size()).isEqualTo(3); - - long totalRows = inputShards.stream() - .mapToLong(ShardInfo::getRowCount) - .sum(); - long expectedOutputShards = computeExpectedOutputShards(totalRows); - - Set inputUuids = inputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); - - long transactionId = 1; - List outputShards = compactor.compact(transactionId, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes)); - assertThat(outputShards.size()).isEqualTo(expectedOutputShards); - - Set outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); - assertShardEqualsIgnoreOrder(inputUuids, outputUuids, columnIds, columnTypes); - } - - @Test - public void testShardCompactorSorted() - throws Exception - { - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DATE, TIMESTAMP_MILLIS, DOUBLE); - List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); - List sortColumnIds = ImmutableList.of(1L, 2L, 3L, 5L, 7L); - List sortOrders = nCopies(sortColumnIds.size(), ASC_NULLS_FIRST); - List sortIndexes = sortColumnIds.stream() - .map(columnIds::indexOf) - .collect(toList()); - - List inputShards = createSortedShards(storageManager, columnIds, columnTypes, sortIndexes, sortOrders, 2); - assertThat(inputShards.size()).isEqualTo(2); - - long totalRows = inputShards.stream().mapToLong(ShardInfo::getRowCount).sum(); - long expectedOutputShards = computeExpectedOutputShards(totalRows); - - Set inputUuids = inputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); - - long transactionId = 1; - List outputShards = compactor.compactSorted(transactionId, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes), sortColumnIds, sortOrders); - List outputUuids = outputShards.stream() - .map(ShardInfo::getShardUuid) - .collect(toList()); - assertThat(outputShards.size()).isEqualTo(expectedOutputShards); - - assertShardEqualsSorted(inputUuids, outputUuids, columnIds, columnTypes, sortIndexes, sortOrders); - } - - private static long computeExpectedOutputShards(long totalRows) - { - return ((totalRows % MAX_SHARD_ROWS) != 0) ? ((totalRows / MAX_SHARD_ROWS) + 1) : (totalRows / MAX_SHARD_ROWS); - } - - private void assertShardEqualsIgnoreOrder(Set inputUuids, Set outputUuids, List columnIds, List columnTypes) - throws IOException - { - MaterializedResult inputRows = getMaterializedRows(ImmutableList.copyOf(inputUuids), columnIds, columnTypes); - MaterializedResult outputRows = getMaterializedRows(ImmutableList.copyOf(outputUuids), columnIds, columnTypes); - - assertEqualsIgnoreOrder(outputRows, inputRows); - } - - private void assertShardEqualsSorted(Set inputUuids, List outputUuids, List columnIds, List columnTypes, List sortIndexes, List sortOrders) - throws IOException - { - List inputPages = getPages(inputUuids, columnIds, columnTypes); - List sortTypes = sortIndexes.stream().map(columnTypes::get).collect(toList()); - - MaterializedResult inputRowsSorted = sortAndMaterialize(inputPages, columnTypes, sortIndexes, sortOrders, sortTypes); - MaterializedResult outputRows = extractColumns(getMaterializedRows(outputUuids, columnIds, columnTypes), sortIndexes, sortTypes); - - assertThat(outputRows).containsExactlyElementsOf(inputRowsSorted); - } - - private static MaterializedResult extractColumns(MaterializedResult materializedRows, List indexes, List types) - { - ImmutableList.Builder rows = ImmutableList.builder(); - for (MaterializedRow row : materializedRows) { - Object[] values = new Object[indexes.size()]; - for (int i = 0; i < indexes.size(); i++) { - values[i] = row.getField(indexes.get(i)); - } - rows.add(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, values)); - } - return new MaterializedResult(rows.build(), types); - } - - private static MaterializedResult sortAndMaterialize(List pages, List columnTypes, List sortIndexes, List sortOrders, List sortTypes) - { - long[] orderedAddresses = PAGE_SORTER.sort(columnTypes, pages, sortIndexes, sortOrders, 10_000); - - PageBuilder pageBuilder = new PageBuilder(columnTypes); - for (long orderedAddress : orderedAddresses) { - int pageIndex = PAGE_SORTER.decodePageIndex(orderedAddress); - int positionIndex = PAGE_SORTER.decodePositionIndex(orderedAddress); - - Page page = pages.get(pageIndex); - pageBuilder.declarePosition(); - for (int i = 0; i < columnTypes.size(); i++) { - columnTypes.get(i).appendTo(page.getBlock(i), positionIndex, pageBuilder.getBlockBuilder(i)); - } - } - - // extract the sortIndexes and reorder the blocks by sort indexes (useful for debugging) - Page buildPage = pageBuilder.build(); - Block[] outputBlocks = new Block[buildPage.getChannelCount()]; - - for (int i = 0; i < sortIndexes.size(); i++) { - outputBlocks[i] = buildPage.getBlock(sortIndexes.get(i)); - } - - MaterializedResult.Builder resultBuilder = MaterializedResult.resultBuilder(SESSION, sortTypes); - resultBuilder.page(new Page(outputBlocks)); - - return resultBuilder.build(); - } - - private List getPages(Set uuids, List columnIds, List columnTypes) - throws IOException - { - ImmutableList.Builder pages = ImmutableList.builder(); - for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = getPageSource(columnIds, columnTypes, uuid)) { - while (!pageSource.isFinished()) { - Page outputPage = pageSource.getNextPage(); - if (outputPage == null) { - break; - } - pages.add(outputPage.getLoadedPage()); - } - } - } - return pages.build(); - } - - private MaterializedResult getMaterializedRows(List uuids, List columnIds, List columnTypes) - throws IOException - { - MaterializedResult.Builder rows = MaterializedResult.resultBuilder(SESSION, columnTypes); - for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = getPageSource(columnIds, columnTypes, uuid)) { - MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); - rows.rows(result.getMaterializedRows()); - } - } - return rows.build(); - } - - private ConnectorPageSource getPageSource(List columnIds, List columnTypes, UUID uuid) - { - return storageManager.getPageSource(uuid, OptionalInt.empty(), columnIds, columnTypes, TupleDomain.all(), READER_OPTIONS); - } - - private static List createSortedShards(StorageManager storageManager, List columnIds, List columnTypes, List sortChannels, List sortOrders, int shardCount) - { - StoragePageSink sink = createStoragePageSink(storageManager, columnIds, columnTypes); - for (int shardNum = 0; shardNum < shardCount; shardNum++) { - createSortedShard(columnTypes, sortChannels, sortOrders, sink); - } - return getFutureValue(sink.commit()); - } - - private static void createSortedShard(List columnTypes, List sortChannels, List sortOrders, StoragePageSink sink) - { - List pages = createPages(columnTypes); - - // Sort pages - long[] orderedAddresses = PAGE_SORTER.sort(columnTypes, pages, sortChannels, sortOrders, 10_000); - int[] orderedPageIndex = new int[orderedAddresses.length]; - int[] orderedPositionIndex = new int[orderedAddresses.length]; - - for (int i = 0; i < orderedAddresses.length; i++) { - orderedPageIndex[i] = PAGE_SORTER.decodePageIndex(orderedAddresses[i]); - orderedPositionIndex[i] = PAGE_SORTER.decodePositionIndex(orderedAddresses[i]); - } - - // Append sorted pages - sink.appendPages(pages, orderedPageIndex, orderedPositionIndex); - sink.flush(); - } - - private static List createShards(StorageManager storageManager, List columnIds, List columnTypes, int shardCount) - { - StoragePageSink sink = createStoragePageSink(storageManager, columnIds, columnTypes); - for (int i = 0; i < shardCount; i++) { - sink.appendPages(createPages(columnTypes)); - sink.flush(); - } - return getFutureValue(sink.commit()); - } - - private static StoragePageSink createStoragePageSink(StorageManager manager, List columnIds, List columnTypes) - { - long transactionId = 1; - return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); - } - - private static List createPages(List columnTypes) - { - // Creates 10 pages with 10 rows each - int rowCount = 10; - int pageCount = 10; - - // some random values to start off the blocks - int[][] initialValues = {{17, 15, 16, 18, 14}, {59, 55, 54, 53, 58}}; - - ImmutableList.Builder pages = ImmutableList.builder(); - for (int i = 0; i < pageCount; i++) { - pages.add(SequencePageBuilder.createSequencePage(columnTypes, rowCount, initialValues[i % 2])); - } - return pages.build(); - } - - private static List getColumnInfo(List columnIds, List columnTypes) - { - ImmutableList.Builder columnInfos = ImmutableList.builder(); - for (int i = 0; i < columnIds.size(); i++) { - columnInfos.add(new ColumnInfo(columnIds.get(i), columnTypes.get(i))); - } - return columnInfos.build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizationManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizationManager.java deleted file mode 100644 index 7ee6ae0061a6..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizationManager.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import io.airlift.units.Duration; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizationManager.createOrganizationSets; -import static io.trino.plugin.raptor.legacy.storage.organization.TestCompactionSetCreator.extractIndexes; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.stream.Collectors.toSet; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardOrganizationManager -{ - private Jdbi dbi; - private Handle dummyHandle; - private MetadataDao metadataDao; - private ShardOrganizerDao organizerDao; - - private static final Table tableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.empty(), true); - private static final Table temporalTableInfo = new Table(1L, Optional.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.of(1), true); - - private static final List types = ImmutableList.of(BIGINT, VARCHAR, DATE, TIMESTAMP_MILLIS); - - @BeforeEach - public void setup() - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - metadataDao = dbi.onDemand(MetadataDao.class); - organizerDao = dbi.onDemand(ShardOrganizerDao.class); - - createTablesWithRetry(dbi); - } - - @AfterEach - public void teardown() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testOrganizationEligibleTables() - { - long table1 = metadataDao.insertTable("schema", "table1", false, true, null, 0); - metadataDao.insertColumn(table1, 1, "foo", 1, "bigint", 1, null); - - metadataDao.insertTable("schema", "table2", false, true, null, 0); - metadataDao.insertTable("schema", "table3", false, false, null, 0); - assertThat(metadataDao.getOrganizationEligibleTables()).isEqualTo(ImmutableSet.of(table1)); - } - - @Test - public void testTableDiscovery() - throws Exception - { - long table1 = metadataDao.insertTable("schema", "table1", false, true, null, 0); - metadataDao.insertColumn(table1, 1, "foo", 1, "bigint", 1, null); - - long table2 = metadataDao.insertTable("schema", "table2", false, true, null, 0); - metadataDao.insertColumn(table2, 1, "foo", 1, "bigint", 1, null); - - metadataDao.insertTable("schema", "table3", false, false, null, 0); - - long intervalMillis = 100; - ShardOrganizationManager organizationManager = createShardOrganizationManager(intervalMillis); - - // initializes tables - Set actual = organizationManager.discoverAndInitializeTablesToOrganize(); - assertThat(actual).isEqualTo(ImmutableSet.of(table1, table2)); - - // update the start times and test that the tables are discovered after interval seconds - long updateTime = System.currentTimeMillis(); - organizerDao.updateLastStartTime("node1", table1, updateTime); - organizerDao.updateLastStartTime("node1", table2, updateTime); - - // wait for some time (interval time) for the tables to be eligible for organization - long start = System.nanoTime(); - while (organizationManager.discoverAndInitializeTablesToOrganize().isEmpty() && - nanosSince(start).toMillis() < intervalMillis + 1000) { - MILLISECONDS.sleep(10); - } - assertThat(organizationManager.discoverAndInitializeTablesToOrganize()).isEqualTo(ImmutableSet.of(table1, table2)); - } - - @Test - public void testSimple() - { - long timestamp = 1L; - int day = 1; - - List shards = ImmutableList.of( - shardWithSortRange(1, ShardRange.of(new Tuple(types, 5L, "hello", day, timestamp), new Tuple(types, 10L, "hello", day, timestamp))), - shardWithSortRange(1, ShardRange.of(new Tuple(types, 7L, "hello", day, timestamp), new Tuple(types, 10L, "hello", day, timestamp))), - shardWithSortRange(1, ShardRange.of(new Tuple(types, 6L, "hello", day, timestamp), new Tuple(types, 9L, "hello", day, timestamp))), - shardWithSortRange(1, ShardRange.of(new Tuple(types, 1L, "hello", day, timestamp), new Tuple(types, 5L, "hello", day, timestamp)))); - Set actual = createOrganizationSets(tableInfo, shards); - - assertThat(actual.size()).isEqualTo(1); - // Shards 0, 1 and 2 are overlapping, so we should get an organization set with these shards - assertThat(getOnlyElement(actual).getShards()).isEqualTo(extractIndexes(shards, 0, 1, 2)); - } - - @Test - public void testSimpleTemporal() - { - List temporalType = ImmutableList.of(DATE); - List types = ImmutableList.of(BIGINT); - - int day1 = 1; - int day2 = 2; - int day4 = 4; - int day5 = 5; - - List shards = ImmutableList.of( - shardWithTemporalRange(1, ShardRange.of(new Tuple(types, 5L), new Tuple(types, 10L)), ShardRange.of(new Tuple(temporalType, day1), new Tuple(temporalType, day2))), - shardWithTemporalRange(1, ShardRange.of(new Tuple(types, 7L), new Tuple(types, 10L)), ShardRange.of(new Tuple(temporalType, day4), new Tuple(temporalType, day5))), - shardWithTemporalRange(1, ShardRange.of(new Tuple(types, 6L), new Tuple(types, 9L)), ShardRange.of(new Tuple(temporalType, day1), new Tuple(temporalType, day2))), - shardWithTemporalRange(1, ShardRange.of(new Tuple(types, 4L), new Tuple(types, 8L)), ShardRange.of(new Tuple(temporalType, day4), new Tuple(temporalType, day5)))); - - Set organizationSets = createOrganizationSets(temporalTableInfo, shards); - Set> actual = organizationSets.stream() - .map(OrganizationSet::getShards) - .collect(toSet()); - - // expect 2 organization sets, of overlapping shards (0, 2) and (1, 3) - assertThat(organizationSets.size()).isEqualTo(2); - assertThat(actual).isEqualTo(ImmutableSet.of(extractIndexes(shards, 0, 2), extractIndexes(shards, 1, 3))); - } - - private static ShardIndexInfo shardWithSortRange(int bucketNumber, ShardRange sortRange) - { - return new ShardIndexInfo( - 1, - OptionalInt.of(bucketNumber), - UUID.randomUUID(), - 1, - 1, - Optional.of(sortRange), - Optional.empty()); - } - - private static ShardIndexInfo shardWithTemporalRange(int bucketNumber, ShardRange sortRange, ShardRange temporalRange) - { - return new ShardIndexInfo( - 1, - OptionalInt.of(bucketNumber), - UUID.randomUUID(), - 1, - 1, - Optional.of(sortRange), - Optional.of(temporalRange)); - } - - private ShardOrganizationManager createShardOrganizationManager(long intervalMillis) - { - return new ShardOrganizationManager(dbi, - "node1", - createShardManager(dbi), - createShardOrganizer(), - true, - new Duration(intervalMillis, MILLISECONDS), - new Duration(5, MINUTES)); - } - - private static class MockJobFactory - implements JobFactory - { - @Override - public Runnable create(OrganizationSet organizationSet) - { - return () -> sleepUninterruptibly(10, MILLISECONDS); - } - } - - private static ShardOrganizer createShardOrganizer() - { - return new ShardOrganizer(new MockJobFactory(), 1); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizer.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizer.java deleted file mode 100644 index f72f4f464a55..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizer.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableSet; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Iterables.getOnlyElement; -import static com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestShardOrganizer -{ - @Test - @Timeout(5) - public void testShardOrganizerInProgress() - throws Exception - { - CountDownLatch canComplete = new CountDownLatch(1); - ShardOrganizer organizer = new ShardOrganizer( - organizationSet -> () -> checkState(awaitUninterruptibly(canComplete, 10, SECONDS)), - 1); - - Set shards = ImmutableSet.of(UUID.randomUUID()); - OrganizationSet organizationSet = new OrganizationSet(1L, shards, OptionalInt.empty()); - - organizer.enqueue(organizationSet); - - assertThat(organizer.inProgress(getOnlyElement(shards))).isTrue(); - assertThat(organizer.getShardsInProgress()).isEqualTo(1); - - canComplete.countDown(); - while (organizer.inProgress(getOnlyElement(shards))) { - MILLISECONDS.sleep(10); - } - assertThat(organizer.inProgress(getOnlyElement(shards))).isFalse(); - assertThat(organizer.getShardsInProgress()).isEqualTo(0); - organizer.shutdown(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizerUtil.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizerUtil.java deleted file mode 100644 index fcb49f441cb0..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardOrganizerUtil.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import io.trino.plugin.raptor.legacy.RaptorMetadata; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.ColumnStats; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.plugin.raptor.legacy.metadata.ShardMetadata; -import io.trino.plugin.raptor.legacy.metadata.Table; -import io.trino.plugin.raptor.legacy.metadata.TableColumn; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.Type; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.nio.file.Files; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static com.google.common.collect.MoreCollectors.onlyElement; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.shardInfo; -import static io.trino.plugin.raptor.legacy.storage.organization.ShardOrganizerUtil.getOrganizationEligibleShards; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardOrganizerUtil -{ - private static final List COLUMNS = ImmutableList.of( - new ColumnInfo(1, TIMESTAMP_MILLIS), - new ColumnInfo(2, BIGINT), - new ColumnInfo(3, VARCHAR)); - - private Jdbi dbi; - private Handle dummyHandle; - private File dataDir; - private ShardManager shardManager; - private MetadataDao metadataDao; - private ConnectorMetadata metadata; - - @BeforeEach - public void setup() - throws Exception - { - dbi = createTestingJdbi(); - dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - dataDir = Files.createTempDirectory(null).toFile(); - - metadata = new RaptorMetadata(dbi, createShardManager(dbi)); - - metadataDao = dbi.onDemand(MetadataDao.class); - shardManager = createShardManager(dbi); - } - - @AfterEach - public void teardown() - throws Exception - { - dummyHandle.close(); - dummyHandle = null; - deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); - } - - @Test - public void testGetOrganizationEligibleShards() - { - int day1 = 1111; - int day2 = 2222; - - SchemaTableName tableName = new SchemaTableName("default", "test"); - metadata.createTable(SESSION, tableMetadataBuilder(tableName) - .column("orderkey", BIGINT) - .column("orderdate", DATE) - .column("orderstatus", createVarcharType(3)) - .property("ordering", ImmutableList.of("orderstatus", "orderkey")) - .property("temporal_column", "orderdate") - .build(), - SaveMode.FAIL); - Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName()); - List tableColumns = metadataDao.listTableColumns(tableInfo.getTableId()); - Map tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName); - - long orderDate = tableColumnMap.get("orderdate").getColumnId(); - long orderKey = tableColumnMap.get("orderkey").getColumnId(); - long orderStatus = tableColumnMap.get("orderstatus").getColumnId(); - - List shards = ImmutableList.builder() - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day1, day1 + 10), - new ColumnStats(orderKey, 13L, 14L), - new ColumnStats(orderStatus, "aaa", "abc")))) - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day2, day2 + 100), - new ColumnStats(orderKey, 2L, 20L), - new ColumnStats(orderStatus, "aaa", "abc")))) - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day1, day2), - new ColumnStats(orderKey, 2L, 11L), - new ColumnStats(orderStatus, "aaa", "abc")))) - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day1, day2), - new ColumnStats(orderKey, 2L, null), - new ColumnStats(orderStatus, "aaa", "abc")))) - .add(shardInfo( - UUID.randomUUID(), - "node1", - ImmutableList.of( - new ColumnStats(orderDate, day1, null), - new ColumnStats(orderKey, 2L, 11L), - new ColumnStats(orderStatus, "aaa", "abc")))) - .build(); - - long transactionId = shardManager.beginTransaction(); - shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0); - Set shardMetadatas = shardManager.getNodeShards("node1"); - - Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId()); - TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId); - - Set actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false)); - List expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty()); - - assertThat(actual).containsExactlyElementsOf(expected); - - List sortColumns = metadataDao.listSortColumns(tableInfo.getTableId()); - Set actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true)); - List expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns)); - - assertThat(actualSortRange).containsExactlyElementsOf(expectedSortRange); - } - - private static List getShardIndexInfo(Table tableInfo, List shards, TableColumn temporalColumn, Optional> sortColumns) - { - long tableId = tableInfo.getTableId(); - Type temporalType = temporalColumn.getDataType(); - - ImmutableList.Builder builder = ImmutableList.builder(); - for (ShardInfo shard : shards) { - ColumnStats temporalColumnStats = shard.getColumnStats().stream() - .filter(columnStats -> columnStats.getColumnId() == temporalColumn.getColumnId()) - .collect(onlyElement()); - - if (temporalColumnStats.getMin() == null || temporalColumnStats.getMax() == null) { - continue; - } - - Optional sortRange = Optional.empty(); - if (sortColumns.isPresent()) { - Map columnIdToStats = Maps.uniqueIndex(shard.getColumnStats(), ColumnStats::getColumnId); - ImmutableList.Builder typesBuilder = ImmutableList.builder(); - ImmutableList.Builder minBuilder = ImmutableList.builder(); - ImmutableList.Builder maxBuilder = ImmutableList.builder(); - boolean isShardEligible = true; - for (TableColumn sortColumn : sortColumns.get()) { - ColumnStats columnStats = columnIdToStats.get(sortColumn.getColumnId()); - typesBuilder.add(sortColumn.getDataType()); - - if (columnStats.getMin() == null || columnStats.getMax() == null) { - isShardEligible = false; - break; - } - - minBuilder.add(columnStats.getMin()); - maxBuilder.add(columnStats.getMax()); - } - - if (!isShardEligible) { - continue; - } - - List types = typesBuilder.build(); - List minValues = minBuilder.build(); - List maxValues = maxBuilder.build(); - sortRange = Optional.of(ShardRange.of(new Tuple(types, minValues), new Tuple(types, maxValues))); - } - builder.add(new ShardIndexInfo( - tableId, - OptionalInt.empty(), - shard.getShardUuid(), - shard.getRowCount(), - shard.getUncompressedSize(), - sortRange, - Optional.of(ShardRange.of( - new Tuple(temporalType, temporalColumnStats.getMin()), - new Tuple(temporalType, temporalColumnStats.getMax()))))); - } - return builder.build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardRange.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardRange.java deleted file mode 100644 index a85d24c2cb23..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardRange.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestShardRange -{ - @Test - public void testEnclosesIsSymmetric() - { - List types = ImmutableList.of(BIGINT, VARCHAR, BOOLEAN, TIMESTAMP_MILLIS); - ShardRange range = ShardRange.of(new Tuple(types, 2L, "aaa", true, 1L), new Tuple(types, 5L, "ccc", false, 2L)); - assertThat(range.encloses(range)).isTrue(); - } - - @Test - public void testEnclosingRange() - { - List types1 = ImmutableList.of(BIGINT); - ShardRange range1 = ShardRange.of(new Tuple(types1, 2L), new Tuple(types1, 5L)); - - ShardRange enclosesRange1 = ShardRange.of(new Tuple(types1, 1L), new Tuple(types1, 10L)); - ShardRange notEnclosesRange1 = ShardRange.of(new Tuple(types1, 1L), new Tuple(types1, 4L)); - - assertThat(enclosesRange1.encloses(range1)).isTrue(); - assertThat(notEnclosesRange1.encloses(range1)).isFalse(); - - List types2 = ImmutableList.of(BIGINT, VARCHAR); - ShardRange range2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "ccc")); - ShardRange enclosesRange2 = ShardRange.of(new Tuple(types2, 1L, "ccc"), new Tuple(types2, 10L, "ccc")); - ShardRange notEnclosesRange2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "bbb")); - - assertThat(range2.encloses(range2)).isTrue(); - assertThat(enclosesRange2.encloses(range2)).isTrue(); - assertThat(notEnclosesRange2.encloses(range2)).isFalse(); - } - - @Test - public void testOverlapsIsSymmetric() - { - List types = ImmutableList.of(BIGINT, VARCHAR, BOOLEAN, TIMESTAMP_MILLIS); - ShardRange range = ShardRange.of(new Tuple(types, 2L, "aaa", true, 1L), new Tuple(types, 5L, "ccc", false, 2L)); - assertThat(range.overlaps(range)).isTrue(); - } - - @Test - public void testOverlappingRange() - { - List types1 = ImmutableList.of(BIGINT); - ShardRange range1 = ShardRange.of(new Tuple(types1, 2L), new Tuple(types1, 5L)); - - ShardRange enclosesRange1 = ShardRange.of(new Tuple(types1, 1L), new Tuple(types1, 10L)); - ShardRange overlapsRange1 = ShardRange.of(new Tuple(types1, 1L), new Tuple(types1, 4L)); - ShardRange notOverlapsRange1 = ShardRange.of(new Tuple(types1, 6L), new Tuple(types1, 8L)); - - assertThat(enclosesRange1.overlaps(range1)).isTrue(); - assertThat(overlapsRange1.overlaps(range1)).isTrue(); - assertThat(notOverlapsRange1.overlaps(range1)).isFalse(); - - List types2 = ImmutableList.of(BIGINT, VARCHAR); - ShardRange range2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "ccc")); - ShardRange enclosesRange2 = ShardRange.of(new Tuple(types2, 1L, "ccc"), new Tuple(types2, 10L, "ccc")); - ShardRange overlapsRange2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "bbb")); - ShardRange notOverlapsRange2 = ShardRange.of(new Tuple(types2, 6L, "aaa"), new Tuple(types2, 8L, "bbb")); - - assertThat(enclosesRange2.encloses(range2)).isTrue(); - assertThat(overlapsRange2.overlaps(range2)).isTrue(); - assertThat(notOverlapsRange2.overlaps(range2)).isFalse(); - } - - @Test - public void testAdjacentRange() - { - List types1 = ImmutableList.of(BIGINT); - ShardRange range1 = ShardRange.of(new Tuple(types1, 2L), new Tuple(types1, 5L)); - ShardRange adjacentRange1 = ShardRange.of(new Tuple(types1, 5L), new Tuple(types1, 10L)); - - assertThat(range1.adjacent(range1)).isFalse(); - - assertThat(adjacentRange1.adjacent(range1)).isTrue(); - assertThat(range1.adjacent(adjacentRange1)).isTrue(); - - List types2 = ImmutableList.of(BIGINT, VARCHAR); - ShardRange range2 = ShardRange.of(new Tuple(types2, 2L, "aaa"), new Tuple(types2, 5L, "ccc")); - ShardRange adjacentRange2 = ShardRange.of(new Tuple(types2, 5L, "ccc"), new Tuple(types2, 10L, "ccc")); - ShardRange subsetAdjacentRange2 = ShardRange.of(new Tuple(types2, 5L, "ddd"), new Tuple(types2, 10L, "ccc")); - ShardRange overlapsRange2 = ShardRange.of(new Tuple(types2, 3L, "aaa"), new Tuple(types2, 10L, "ccc")); - ShardRange notAdjacentRange2 = ShardRange.of(new Tuple(types2, 6L, "ccc"), new Tuple(types2, 10L, "ccc")); - - assertThat(adjacentRange2.adjacent(range2)).isTrue(); - assertThat(subsetAdjacentRange2.adjacent(range2)).isTrue(); - assertThat(overlapsRange2.adjacent(range2)).isFalse(); - assertThat(notAdjacentRange2.adjacent(range2)).isFalse(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTemporalFunction.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTemporalFunction.java deleted file mode 100644 index e294f9f51d8a..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTemporalFunction.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import org.joda.time.DateTime; -import org.junit.jupiter.api.Test; - -import java.time.Duration; - -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.type.DateTimes.MICROSECONDS_PER_MILLISECOND; -import static org.assertj.core.api.Assertions.assertThat; -import static org.joda.time.DateTimeZone.UTC; - -public class TestTemporalFunction -{ - private static final DateTime DATE_TIME = new DateTime(1970, 1, 2, 0, 0, 0, UTC); - - @Test - public void testDateBlock() - { - BlockBuilder blockBuilder = DATE.createBlockBuilder(null, 2); - DATE.writeLong(blockBuilder, 13); - DATE.writeLong(blockBuilder, 42); - Block block = blockBuilder.build(); - - assertThat(TemporalFunction.getDay(DATE, block, 0)).isEqualTo(13); - assertThat(TemporalFunction.getDay(DATE, block, 1)).isEqualTo(42); - } - - @Test - public void testTimestampBlock() - { - BlockBuilder blockBuilder = TIMESTAMP_MILLIS.createBlockBuilder(null, 4); - - // start and end of UTC day - TIMESTAMP_MILLIS.writeLong(blockBuilder, DATE_TIME.getMillis() * MICROSECONDS_PER_MILLISECOND); - TIMESTAMP_MILLIS.writeLong(blockBuilder, (DATE_TIME.getMillis() + Duration.ofHours(23).toMillis()) * MICROSECONDS_PER_MILLISECOND); - - Block block = blockBuilder.build(); - - assertThat(TemporalFunction.getDay(TIMESTAMP_MILLIS, block, 0)).isEqualTo(1); - assertThat(TemporalFunction.getDay(TIMESTAMP_MILLIS, block, 1)).isEqualTo(1); - } - - @Test - public void testDateShardRange() - { - assertThat(TemporalFunction.getDayFromRange(dateRange(2, 2))).isEqualTo(2); - assertThat(TemporalFunction.getDayFromRange(dateRange(13, 13))).isEqualTo(13); - - // date is determined from lowest shard - assertThat(TemporalFunction.getDayFromRange(dateRange(2, 5))).isEqualTo(2); - } - - @Test - public void testTimestampShardRange() - { - // The time frame should be look like following: - - // time range covers full day of day 1 - assertThat(TemporalFunction.getDayFromRange(timeRange(DATE_TIME.getMillis(), Duration.ofDays(1)))).isEqualTo(1); - // time range covers full day of day 1 and 2 - assertThat(TemporalFunction.getDayFromRange(timeRange(DATE_TIME.getMillis(), Duration.ofDays(2)))).isEqualTo(2); - // time range covers 13 hours of day 1 and 11 hours of day 2 - assertThat(TemporalFunction.getDayFromRange(timeRange(DATE_TIME.getMillis() + Duration.ofHours(11).toMillis(), Duration.ofHours(24)))).isEqualTo(1); - // time range covers 11 hours of day 0 and 13 hours of day 1 - assertThat(TemporalFunction.getDayFromRange(timeRange(DATE_TIME.getMillis() + Duration.ofHours(13).toMillis(), Duration.ofHours(24)))).isEqualTo(2); - } - - private static ShardRange dateRange(int start, int end) - { - return ShardRange.of(new Tuple(DATE, start), new Tuple(DATE, end)); - } - - private static ShardRange timeRange(long start, Duration duration) - { - return ShardRange.of( - new Tuple(TIMESTAMP_MILLIS, start), - new Tuple(TIMESTAMP_MILLIS, start + duration.toMillis())); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTuple.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTuple.java deleted file mode 100644 index 1fc358fb2297..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestTuple.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.storage.organization; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThan; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class TestTuple -{ - @Test - public void testComparableTuple() - { - List types = ImmutableList.of(BIGINT, VARCHAR, BOOLEAN, DOUBLE, DATE, TIMESTAMP_MILLIS); - - Tuple tuple1 = new Tuple(types, ImmutableList.of(1L, "hello", false, 1.2d, 11111, 1112)); - Tuple equalToTuple1 = new Tuple(types, ImmutableList.of(1L, "hello", false, 1.2d, 11111, 1112)); - Tuple greaterThanTuple1 = new Tuple(types, ImmutableList.of(1L, "hello", false, 1.2d, 11111, 1113)); - Tuple lessThanTuple1 = new Tuple(types, ImmutableList.of(1L, "hello", false, 1.2d, 11111, 1111)); - - assertThat(tuple1.compareTo(equalToTuple1)).isEqualTo(0); - assertLessThan(tuple1.compareTo(greaterThanTuple1), 0); - assertGreaterThan(tuple1.compareTo(lessThanTuple1), 0); - } - - @Test - public void testMismatchedTypes() - { - assertThatThrownBy(() -> { - List types1 = ImmutableList.of(createVarcharType(3)); - List types2 = ImmutableList.of(createVarcharType(4)); - Tuple tuple1 = new Tuple(types1, ImmutableList.of("abc")); - Tuple tuple2 = new Tuple(types2, ImmutableList.of("abcd")); - tuple1.compareTo(tuple2); - }) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("types must be the same"); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/systemtables/TestShardMetadataRecordCursor.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/systemtables/TestShardMetadataRecordCursor.java deleted file mode 100644 index d00110f58d4f..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/systemtables/TestShardMetadataRecordCursor.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.systemtables; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.airlift.slice.Slice; -import io.trino.plugin.raptor.legacy.RaptorMetadata; -import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; -import io.trino.plugin.raptor.legacy.metadata.MetadataDao; -import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.metadata.ShardManager; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.predicate.ValueSet; -import io.trino.spi.type.Type; -import io.trino.testing.MaterializedRow; -import org.jdbi.v3.core.Handle; -import org.jdbi.v3.core.Jdbi; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.UUID; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; -import static io.trino.plugin.raptor.legacy.DatabaseTesting.createTestingJdbi; -import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.systemtables.ShardMetadataRecordCursor.SHARD_METADATA; -import static io.trino.spi.predicate.Range.greaterThan; -import static io.trino.spi.predicate.Range.lessThanOrEqual; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static io.trino.testing.MaterializedResult.DEFAULT_PRECISION; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestShardMetadataRecordCursor -{ - private static final SchemaTableName DEFAULT_TEST_ORDERS = new SchemaTableName("test", "orders"); - - private Handle dummyHandle; - private ConnectorMetadata metadata; - private Jdbi dbi; - - @BeforeEach - public void setup() - { - dbi = createTestingJdbi(); - this.dummyHandle = dbi.open(); - createTablesWithRetry(dbi); - this.metadata = new RaptorMetadata(dbi, createShardManager(dbi)); - - // Create table - ConnectorTableMetadata table = tableMetadataBuilder(DEFAULT_TEST_ORDERS) - .column("orderkey", BIGINT) - .column("orderdate", DATE) - .property("temporal_column", "orderdate") - .build(); - createTable(table); - } - - @AfterEach - public void teardown() - { - dummyHandle.close(); - dummyHandle = null; - } - - @Test - public void testSimple() - { - ShardManager shardManager = createShardManager(dbi); - - // Add shards to the table - long tableId = 1; - OptionalInt bucketNumber = OptionalInt.empty(); - UUID uuid1 = UUID.randomUUID(); - UUID uuid2 = UUID.randomUUID(); - UUID uuid3 = UUID.randomUUID(); - ShardInfo shardInfo1 = new ShardInfo(uuid1, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 1, 10, 100, 0x1234); - ShardInfo shardInfo2 = new ShardInfo(uuid2, bucketNumber, ImmutableSet.of("node2"), ImmutableList.of(), 2, 20, 200, 0xCAFEBABEDEADBEEFL); - ShardInfo shardInfo3 = new ShardInfo(uuid3, bucketNumber, ImmutableSet.of("node3"), ImmutableList.of(), 3, 30, 300, 0xFEDCBA0987654321L); - List shards = ImmutableList.of(shardInfo1, shardInfo2, shardInfo3); - - long transactionId = shardManager.beginTransaction(); - - shardManager.commitShards( - transactionId, - tableId, - ImmutableList.of( - new ColumnInfo(1, BIGINT), - new ColumnInfo(2, DATE)), - shards, - Optional.empty(), - 0); - - Slice schema = utf8Slice(DEFAULT_TEST_ORDERS.getSchemaName()); - Slice table = utf8Slice(DEFAULT_TEST_ORDERS.getTableName()); - - DateTime date1 = DateTime.parse("2015-01-01T00:00"); - DateTime date2 = DateTime.parse("2015-01-02T00:00"); - TupleDomain tupleDomain = TupleDomain.withColumnDomains( - ImmutableMap.builder() - .put(0, Domain.singleValue(createVarcharType(10), schema)) - .put(1, Domain.create(ValueSet.ofRanges(lessThanOrEqual(createVarcharType(10), table)), true)) - .put(8, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)) - .put(9, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)) - .buildOrThrow()); - - List actual; - try (RecordCursor cursor = new ShardMetadataSystemTable(dbi).cursor(null, SESSION, tupleDomain)) { - actual = getMaterializedResults(cursor, SHARD_METADATA.getColumns()); - } - assertThat(actual.size()).isEqualTo(3); - - List expected = ImmutableList.of( - new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid1.toString()), null, 100L, 10L, 1L, utf8Slice("0000000000001234"), null, null, null, null), - new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid2.toString()), null, 200L, 20L, 2L, utf8Slice("cafebabedeadbeef"), null, null, null, null), - new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid3.toString()), null, 300L, 30L, 3L, utf8Slice("fedcba0987654321"), null, null, null, null)); - - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testNoSchemaFilter() - { - // Create "orders" table in a different schema - createTable(tableMetadataBuilder(new SchemaTableName("other", "orders")) - .column("orderkey", BIGINT) - .build()); - - // Create another table that should not be selected - createTable(tableMetadataBuilder(new SchemaTableName("schema1", "foo")) - .column("orderkey", BIGINT) - .build()); - - TupleDomain tupleDomain = TupleDomain.withColumnDomains( - ImmutableMap.of(1, Domain.singleValue(createVarcharType(10), utf8Slice("orders")))); - - MetadataDao metadataDao = dummyHandle.attach(MetadataDao.class); - Set actual = ImmutableSet.copyOf(ShardMetadataRecordCursor.getTableIds(dbi, tupleDomain)); - Set expected = ImmutableSet.of( - metadataDao.getTableInformation("other", "orders").getTableId(), - metadataDao.getTableInformation("test", "orders").getTableId()); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testNoTableFilter() - { - // Create "orders" table in a different schema - createTable(tableMetadataBuilder(new SchemaTableName("test", "orders2")) - .column("orderkey", BIGINT) - .build()); - - // Create another table that should not be selected - createTable(tableMetadataBuilder(new SchemaTableName("schema1", "foo")) - .column("orderkey", BIGINT) - .build()); - - TupleDomain tupleDomain = TupleDomain.withColumnDomains( - ImmutableMap.of(0, Domain.singleValue(createVarcharType(10), utf8Slice("test")))); - - MetadataDao metadataDao = dummyHandle.attach(MetadataDao.class); - Set actual = ImmutableSet.copyOf(ShardMetadataRecordCursor.getTableIds(dbi, tupleDomain)); - Set expected = ImmutableSet.of( - metadataDao.getTableInformation("test", "orders").getTableId(), - metadataDao.getTableInformation("test", "orders2").getTableId()); - assertThat(actual).isEqualTo(expected); - } - - private void createTable(ConnectorTableMetadata table) - { - metadata.createTable(SESSION, table, SaveMode.FAIL); - } - - private static List getMaterializedResults(RecordCursor cursor, List columns) - { - List types = columns.stream().map(ColumnMetadata::getType).collect(toList()); - - ImmutableList.Builder rowBuilder = ImmutableList.builder(); - for (int i = 0; i < types.size(); i++) { - assertThat(cursor.getType(i)).isEqualTo(types.get(i)); - } - - while (cursor.advanceNextPosition()) { - List values = new ArrayList<>(); - for (int i = 0; i < columns.size(); i++) { - Type type = columns.get(i).getType(); - Class javaType = type.getJavaType(); - if (cursor.isNull(i)) { - values.add(null); - } - else if (javaType == boolean.class) { - values.add(cursor.getBoolean(i)); - } - else if (javaType == long.class) { - values.add(cursor.getLong(i)); - } - else if (javaType == double.class) { - values.add(cursor.getDouble(i)); - } - else if (javaType == Slice.class) { - values.add(cursor.getSlice(i)); - } - } - rowBuilder.add(new MaterializedRow(DEFAULT_PRECISION, values)); - } - return rowBuilder.build(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/util/TestPrioritizedFifoExecutor.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/util/TestPrioritizedFifoExecutor.java deleted file mode 100644 index 300764f06b48..000000000000 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/util/TestPrioritizedFifoExecutor.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.raptor.legacy.util; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_METHOD) -@Execution(SAME_THREAD) -public class TestPrioritizedFifoExecutor -{ - private static final Comparator DUMMY_COMPARATOR = (unused, unused2) -> 0; - - private ExecutorService executor; - - @BeforeEach - public void setUp() - { - executor = newCachedThreadPool(); - } - - @AfterEach - public void tearDown() - { - executor.shutdownNow(); - } - - @Test - public void testCounter() - throws Exception - { - // enforce single thread - PrioritizedFifoExecutor executor = new PrioritizedFifoExecutor<>(this.executor, 1, DUMMY_COMPARATOR); - - int totalTasks = 100_000; - AtomicInteger counter = new AtomicInteger(); - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch completeLatch = new CountDownLatch(totalTasks); - List> futures = new ArrayList<>(); - - for (int i = 0; i < totalTasks; i++) { - int taskNumber = i; - futures.add(executor.submit(() -> { - try { - // wait for the go signal - - assertThat(awaitUninterruptibly(startLatch, 1, TimeUnit.MINUTES)).isTrue(); - assertThat(futures.get(taskNumber).isDone()).isFalse(); - - // intentional distinct read and write calls - int initialCount = counter.get(); - counter.set(initialCount + 1); - } - finally { - completeLatch.countDown(); - } - })); - } - - for (Future future : futures) { - assertThat(future.isDone()).isFalse(); - } - - // signal go and wait for tasks to complete - startLatch.countDown(); - assertThat(awaitUninterruptibly(completeLatch, 1, TimeUnit.MINUTES)).isTrue(); - - assertThat(counter.get()).isEqualTo(totalTasks); - // since this is a fifo executor with one thread and completeLatch is decremented inside the future, - // the last future may not be done yet, but all the rest must be - futures.getLast().get(1, TimeUnit.MINUTES); - for (Future future : futures) { - assertThat(future.isDone()).isTrue(); - } - } - - @Test - public void testSingleThreadBound() - { - testBound(1, 100_000); - } - - @Test - public void testDoubleThreadBound() - { - testBound(2, 100_000); - } - - private void testBound(int maxThreads, int totalTasks) - { - PrioritizedFifoExecutor boundedExecutor = new PrioritizedFifoExecutor<>(executor, maxThreads, DUMMY_COMPARATOR); - - AtomicInteger activeThreadCount = new AtomicInteger(); - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch completeLatch = new CountDownLatch(totalTasks); - AtomicBoolean failed = new AtomicBoolean(); - - for (int i = 0; i < totalTasks; i++) { - boundedExecutor.submit(() -> { - try { - // wait for the go signal - awaitUninterruptibly(startLatch); - - int count = activeThreadCount.incrementAndGet(); - if (count < 1 || count > maxThreads) { - failed.set(true); - } - activeThreadCount.decrementAndGet(); - } - finally { - completeLatch.countDown(); - } - }); - } - - // signal go and wait for tasks to complete - startLatch.countDown(); - assertThat(awaitUninterruptibly(completeLatch, 1, TimeUnit.MINUTES)).isTrue(); - - assertThat(failed.get()).isFalse(); - } -} diff --git a/plugin/trino-raptor-legacy/src/test/resources/io/trino/plugin/raptor/legacy/security/security.json b/plugin/trino-raptor-legacy/src/test/resources/io/trino/plugin/raptor/legacy/security/security.json deleted file mode 100644 index dbea06e30017..000000000000 --- a/plugin/trino-raptor-legacy/src/test/resources/io/trino/plugin/raptor/legacy/security/security.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "tables": [ - { - "user": "user", - "privileges": [ - "SELECT", - "OWNERSHIP" - ] - } - ], - "schemas": [ - { - "owner": false - } - ] -} - diff --git a/pom.xml b/pom.xml index d99015685d89..651b783926ba 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,6 @@ plugin/trino-pinot plugin/trino-postgresql plugin/trino-prometheus - plugin/trino-raptor-legacy plugin/trino-redis plugin/trino-redshift plugin/trino-resource-group-managers @@ -1400,12 +1399,6 @@ ${project.version} - - io.trino - trino-raptor-legacy - ${project.version} - - io.trino trino-re2j diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java index ba3dd24027f1..0950b188a0c6 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java @@ -73,7 +73,6 @@ public void extendEnvironment(Environment.Builder builder) "pinot", "postgresql", "prometheus", - "raptor_legacy", "redis", "redshift", "snowflake", diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/raptor_legacy.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/raptor_legacy.properties deleted file mode 100644 index 5e04b760e2ed..000000000000 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/raptor_legacy.properties +++ /dev/null @@ -1,4 +0,0 @@ -connector.name=raptor_legacy -metadata.db.type=h2 -metadata.db.filename=mem:raptor;DB_CLOSE_DELAY=-1 -storage.data-directory=var/data diff --git a/testing/trino-server-dev/etc/catalog/raptor.properties b/testing/trino-server-dev/etc/catalog/raptor.properties deleted file mode 100644 index 08a7d4352a07..000000000000 --- a/testing/trino-server-dev/etc/catalog/raptor.properties +++ /dev/null @@ -1,13 +0,0 @@ -# -# WARNING -# ^^^^^^^ -# This configuration file is for development only and should NOT be used -# in production. For example configuration, see the Trino documentation. -# - -connector.name=raptor_legacy -metadata.db.type=h2 -metadata.db.filename=mem:raptor;DB_CLOSE_DELAY=-1 -#metadata.db.type=mysql -#metadata.db.connections.max=500 -storage.data-directory=var/data diff --git a/testing/trino-server-dev/etc/config.properties b/testing/trino-server-dev/etc/config.properties index 24c61d89a0d3..d9ddf55aa5f8 100644 --- a/testing/trino-server-dev/etc/config.properties +++ b/testing/trino-server-dev/etc/config.properties @@ -34,7 +34,6 @@ plugin.bundles=\ ../../plugin/trino-cassandra/pom.xml,\ ../../plugin/trino-memory/pom.xml,\ ../../plugin/trino-jmx/pom.xml,\ - ../../plugin/trino-raptor-legacy/pom.xml,\ ../../plugin/trino-hive/pom.xml,\ ../../plugin/trino-hudi/pom.xml,\ ../../plugin/trino-example-http/pom.xml,\