From 79536fed03767b4fbcc5b34a93385aaf0f31cf82 Mon Sep 17 00:00:00 2001 From: TungYuChiang <75083792+TungYuChiang@users.noreply.github.com> Date: Tue, 7 Jan 2025 09:55:53 +0800 Subject: [PATCH] [#5779] feat(iceberg): add OSS support for IcebergRESTService docker image (#6096) ### What changes were proposed in this pull request? add OSS support for IcebergRESTService docker image ### Why are the changes needed? Fix: #5779 no ### How was this patch tested? run SQL with access Aliyun OSS data --- .../iceberg-rest-server/iceberg-rest-server-dependency.sh | 7 +++++++ dev/docker/iceberg-rest-server/rewrite_config.py | 7 +++++++ docs/docker-image-details.md | 2 ++ docs/iceberg-rest-service.md | 8 +++++++- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh index 2235313dc09..852b55b0206 100755 --- a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh +++ b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh @@ -38,6 +38,7 @@ cd ${gravitino_home} ./gradlew :bundles:gcp-bundle:jar ./gradlew :bundles:aws-bundle:jar ./gradlew :bundles:azure-bundle:jar +./gradlew :bundles:aliyun-bundle:jar # prepare bundle jar cd ${iceberg_rest_server_dir} @@ -45,6 +46,7 @@ mkdir -p bundles cp ${gravitino_home}/bundles/gcp-bundle/build/libs/gravitino-gcp-bundle-*.jar bundles/ cp ${gravitino_home}/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*.jar bundles/ cp ${gravitino_home}/bundles/azure-bundle/build/libs/gravitino-azure-bundle-*.jar bundles/ +cp ${gravitino_home}/bundles/aliyun-bundle/build/libs/gravitino-aliyun-bundle-*.jar bundles/ iceberg_gcp_bundle="iceberg-gcp-bundle-1.5.2.jar" if [ ! -f "bundles/${iceberg_gcp_bundle}" ]; then @@ -61,6 +63,11 @@ if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then curl -L -s -o bundles/${iceberg_azure_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/1.5.2/${iceberg_azure_bundle} fi +iceberg_aliyun_bundle="iceberg-aliyun-bundle-1.5.2.jar" +if [ ! -f "bundles/${iceberg_aliyun_bundle}" ]; then + curl -L -s -o bundles/${iceberg_aliyun_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aliyun-bundle/1.5.2/${iceberg_aliyun_bundle} +fi + # download jdbc driver curl -L -s -o bundles/sqlite-jdbc-3.42.0.0.jar https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar diff --git a/dev/docker/iceberg-rest-server/rewrite_config.py b/dev/docker/iceberg-rest-server/rewrite_config.py index b10cdb4bfb7..8b9b42a531c 100755 --- a/dev/docker/iceberg-rest-server/rewrite_config.py +++ b/dev/docker/iceberg-rest-server/rewrite_config.py @@ -36,6 +36,13 @@ "GRAVITINO_AZURE_TENANT_ID" : "azure-tenant-id", "GRAVITINO_AZURE_CLIENT_ID" : "azure-client-id", "GRAVITINO_AZURE_CLIENT_SECRET" : "azure-client-secret", + "GRAVITINO_OSS_ACCESS_KEY": "oss-access-key-id", + "GRAVITINO_OSS_SECRET_KEY": "oss-secret-access-key", + "GRAVITINO_OSS_ENDPOINT": "oss-endpoint", + "GRAVITINO_OSS_REGION": "oss-region", + "GRAVITINO_OSS_ROLE_ARN": "oss-role-arn", + "GRAVITINO_OSS_EXTERNAL_ID": "oss-external-id", + } init_config = { diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index c723c009d93..48b3bd191a1 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -59,6 +59,8 @@ docker run --rm -d -p 9001:9001 apache/gravitino-iceberg-rest:0.7.0-incubating ``` Changelog +- apache/gravitino-iceberg-rest:0.8.0-incubating + - Supports OSS and ADLS storage. - apache/gravitino-iceberg-rest:0.7.0-incubating - Using JDBC catalog backend. diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index f21ca35a43a..5adc75ad835 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -441,7 +441,7 @@ SELECT * FROM dml.test; You could run Gravitino Iceberg REST server though docker container: ```shell -docker run -d -p 9001:9001 apache/gravitino-iceberg-rest:0.7.0-incubating +docker run -d -p 9001:9001 apache/gravitino-iceberg-rest:0.8.0-incubating ``` Gravitino Iceberg REST server in docker image could access local storage by default, you could set the following environment variables if the storage is cloud/remote storage like S3, please refer to [storage section](#storage) for more details. @@ -464,6 +464,12 @@ Gravitino Iceberg REST server in docker image could access local storage by defa | `GRAVITINO_AZURE_TENANT_ID` | `gravitino.iceberg-rest.azure-tenant-id` | 0.8.0-incubating | | `GRAVITINO_AZURE_CLIENT_ID` | `gravitino.iceberg-rest.azure-client-id` | 0.8.0-incubating | | `GRAVITINO_AZURE_CLIENT_SECRET` | `gravitino.iceberg-rest.azure-client-secret` | 0.8.0-incubating | +| `GRAVITINO_OSS_ACCESS_KEY` | `gravitino.iceberg-rest.oss-access-key-id` | 0.8.0-incubating | +| `GRAVITINO_OSS_SECRET_KEY` | `gravitino.iceberg-rest.oss-secret-access-key` | 0.8.0-incubating | +| `GRAVITINO_OSS_ENDPOINT` | `gravitino.iceberg-rest.oss-endpoint` | 0.8.0-incubating | +| `GRAVITINO_OSS_REGION` | `gravitino.iceberg-rest.oss-region` | 0.8.0-incubating | +| `GRAVITINO_OSS_ROLE_ARN` | `gravitino.iceberg-rest.oss-role-arn` | 0.8.0-incubating | +| `GRAVITINO_OSS_EXTERNAL_ID` | `gravitino.iceberg-rest.oss-external-id` | 0.8.0-incubating | The below environment is deprecated, please use the corresponding configuration items instead.