Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[regression](cloud) Add missing new operator for recycler plugin #37393

Merged
merged 5 commits into from
Jul 8, 2024
Merged

[regression](cloud) Add missing new operator for recycler plugin #37393

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
db47ef4
add new
ByteYue Jul 7, 2024
fe3caac
fix
ByteYue Jul 7, 2024
282f9bc
ok
ByteYue Jul 7, 2024
e840a1b
t
ByteYue Jul 7, 2024
e728e29
add import
ByteYue Jul 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions regression-test/framework/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -395,5 +395,11 @@ under the License.
<artifactId>azure-storage-blob-batch</artifactId>
<version>12.22.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.104.Final</version>
</dependency>
</dependencies>
</project>
18 changes: 11 additions & 7 deletions ...ion-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectListUtil.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,11 @@ import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.ListBlobsOptions;
import com.azure.storage.common.StorageSharedKeyCredential;

import java.time.Duration;
import java.util.Iterator;

interface ListObjectsFileNames {
public boolean isEmpty(String tableName, String tableId);
public boolean isEmpty(String tableName, String tabletId);
public Set<String> listObjects(String userName, String userId);
};

Expand All @@ -68,7 +69,7 @@ class AwsListObjectsFileNames implements ListObjectsFileNames {
.withCredentials(new AWSStaticCredentialsProvider(credentials)).build()
}

public boolean isEmpty(String tableName, String tableId) {
public boolean isEmpty(String tableName, String tabletId) {
def objectListing = s3Client.listObjects(
new ListObjectsRequest().withMaxKeys(1).withBucketName(bucket).withPrefix("${prefix}/data/${tabletId}/"))

Expand Down Expand Up @@ -124,11 +125,14 @@ class AzureListObjectsFileNames implements ListObjectsFileNames {
this.bucket = bucket;
this.suite = suite;
String uri = String.format(URI_TEMPLATE, this.ak, this.bucket);
StorageSharedKeyCredential cred = new StorageSharedKeyCredential(this.ak, this.sk);
this.containerClient = new BlobContainerClientBuilder().credential(cred).endpoint(uri).build();
StorageSharedKeyCredential cred = new StorageSharedKeyCredential(this.ak, this.sk);
BlobContainerClientBuilder builder = new BlobContainerClientBuilder();
builder.credential(cred);
builder.endpoint(uri);
this.containerClient = builder.buildClient();
}

public boolean isEmpty(String tableName, String tableId) {
public boolean isEmpty(String tableName, String tabletId) {
PagedIterable<BlobItem> blobs = containerClient.listBlobs(
new ListBlobsOptions()
.setPrefix("${prefix}/data/${tabletId}/")
Expand Down Expand Up @@ -173,8 +177,8 @@ class ListObjectsFileNamesUtil {

public static ListObjectsFileNames getListObjectsFileNames(String provider, String ak, String sk, String endpoint, String region, String prefix, String bucket, Suite suite) {
if (provider.equalsIgnoreCase("azure")) {
return AzureListObjectsFileNames(ak, sk, endpoint, region, prefix, bucket, suite)
return new AzureListObjectsFileNames(ak, sk, endpoint, region, prefix, bucket, suite)
}
return AwsListObjectsFileNames(ak, sk, endpoint, region, prefix, bucket, suite)
return new AwsListObjectsFileNames(ak, sk, endpoint, region, prefix, bucket, suite)
}
}
2 changes: 1 addition & 1 deletion regression-test/plugins/cloud_recycler_plugin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ Suite.metaClass.checkRecycleTable = { String token, String instanceId, String cl
// def objectListing = s3Client.listObjects(
// new ListObjectsRequest().withMaxKeys(1).withBucketName(bucket).withPrefix("${prefix}/data/${tabletId}/"))

suite.getLogger().info("tableName: ${tableName}, tabletId:${tabletId}, objectListing:${objectListing.getObjectSummaries()}".toString())
// suite.getLogger().info("tableName: ${tableName}, tabletId:${tabletId}, objectListing:${objectListing.getObjectSummaries()}".toString())
if (!client.isEmpty(tableName, tabletId)) {
return false;
}
Expand Down
Loading