From db47ef4c6c436f4660ce4d5ee5e94dc51f6367e3 Mon Sep 17 00:00:00 2001 From: ByteYue Date: Sun, 7 Jul 2024 18:29:01 +0800 Subject: [PATCH 1/5] add new --- .../org/apache/doris/regression/util/ObjectListUtil.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy index 814db47f378c72..6bd558314b8ed3 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy @@ -173,8 +173,8 @@ class ListObjectsFileNamesUtil { public static ListObjectsFileNames getListObjectsFileNames(String provider, String ak, String sk, String endpoint, String region, String prefix, String bucket, Suite suite) { if (provider.equalsIgnoreCase("azure")) { - return AzureListObjectsFileNames(ak, sk, endpoint, region, prefix, bucket, suite) + return new AzureListObjectsFileNames(ak, sk, endpoint, region, prefix, bucket, suite) } - return AwsListObjectsFileNames(ak, sk, endpoint, region, prefix, bucket, suite) + return new AwsListObjectsFileNames(ak, sk, endpoint, region, prefix, bucket, suite) } } \ No newline at end of file From fe3caac2b9459ab1c49e8d07b58ea6a2913f4427 Mon Sep 17 00:00:00 2001 From: ByteYue Date: Sun, 7 Jul 2024 22:20:58 +0800 Subject: [PATCH 2/5] fix --- .../org/apache/doris/regression/util/ObjectListUtil.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy index 6bd558314b8ed3..738e04e586e288 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy @@ -124,8 +124,11 @@ class AzureListObjectsFileNames implements ListObjectsFileNames { this.bucket = bucket; this.suite = suite; String uri = String.format(URI_TEMPLATE, this.ak, this.bucket); - StorageSharedKeyCredential cred = new StorageSharedKeyCredential(this.ak, this.sk); - this.containerClient = new BlobContainerClientBuilder().credential(cred).endpoint(uri).build(); + StorageSharedKeyCredential cred = new StorageSharedKeyCredential(this.ak, this.sk); + BlobContainerClientBuilder builder = new BlobContainerClientBuilder(); + builder.credential(cred); + builder.endpoint(uri); + this.containerClient = builder.buildClient(); } public boolean isEmpty(String tableName, String tableId) { From 282f9bca45536cba203ae0157c2034762448abfa Mon Sep 17 00:00:00 2001 From: ByteYue Date: Sun, 7 Jul 2024 22:34:45 +0800 Subject: [PATCH 3/5] ok --- regression-test/framework/pom.xml | 6 ++++++ regression-test/plugins/cloud_recycler_plugin.groovy | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml index b689757c5efca0..f50e0fa7cfc882 100644 --- a/regression-test/framework/pom.xml +++ b/regression-test/framework/pom.xml @@ -395,5 +395,11 @@ under the License. azure-storage-blob-batch 12.22.0 + + + io.netty + netty-all + 4.1.104.Final + diff --git a/regression-test/plugins/cloud_recycler_plugin.groovy b/regression-test/plugins/cloud_recycler_plugin.groovy index 0282722d040bee..ed330e94935503 100644 --- a/regression-test/plugins/cloud_recycler_plugin.groovy +++ b/regression-test/plugins/cloud_recycler_plugin.groovy @@ -79,7 +79,7 @@ Suite.metaClass.checkRecycleTable = { String token, String instanceId, String cl // def objectListing = s3Client.listObjects( // new ListObjectsRequest().withMaxKeys(1).withBucketName(bucket).withPrefix("${prefix}/data/${tabletId}/")) - suite.getLogger().info("tableName: ${tableName}, tabletId:${tabletId}, objectListing:${objectListing.getObjectSummaries()}".toString()) + // suite.getLogger().info("tableName: ${tableName}, tabletId:${tabletId}, objectListing:${objectListing.getObjectSummaries()}".toString()) if (!client.isEmpty(tableName, tabletId)) { return false; } From e840a1b353ae34f72ce7989fc37f6939bc497368 Mon Sep 17 00:00:00 2001 From: ByteYue Date: Sun, 7 Jul 2024 22:39:37 +0800 Subject: [PATCH 4/5] t --- .../org/apache/doris/regression/util/ObjectListUtil.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy index 738e04e586e288..0f494485ababe9 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy @@ -41,7 +41,7 @@ import com.azure.storage.common.StorageSharedKeyCredential; import java.util.Iterator; interface ListObjectsFileNames { - public boolean isEmpty(String tableName, String tableId); + public boolean isEmpty(String tableName, String tabletId); public Set listObjects(String userName, String userId); }; @@ -68,7 +68,7 @@ class AwsListObjectsFileNames implements ListObjectsFileNames { .withCredentials(new AWSStaticCredentialsProvider(credentials)).build() } - public boolean isEmpty(String tableName, String tableId) { + public boolean isEmpty(String tableName, String tabletId) { def objectListing = s3Client.listObjects( new ListObjectsRequest().withMaxKeys(1).withBucketName(bucket).withPrefix("${prefix}/data/${tabletId}/")) @@ -131,7 +131,7 @@ class AzureListObjectsFileNames implements ListObjectsFileNames { this.containerClient = builder.buildClient(); } - public boolean isEmpty(String tableName, String tableId) { + public boolean isEmpty(String tableName, String tabletId) { PagedIterable blobs = containerClient.listBlobs( new ListBlobsOptions() .setPrefix("${prefix}/data/${tabletId}/") From e728e29d55b0707520a5dece0d3da2c83d193eea Mon Sep 17 00:00:00 2001 From: ByteYue Date: Sun, 7 Jul 2024 22:43:16 +0800 Subject: [PATCH 5/5] add import --- .../org/apache/doris/regression/util/ObjectListUtil.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy index 0f494485ababe9..7af745f12e5438 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy @@ -38,6 +38,7 @@ import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.ListBlobsOptions; import com.azure.storage.common.StorageSharedKeyCredential; +import java.time.Duration; import java.util.Iterator; interface ListObjectsFileNames {