[#5779] feat(iceberg): add OSS support for IcebergRESTService docker …

…image (#6096) ### What changes were proposed in this pull request? add OSS support for IcebergRESTService docker image ### Why are the changes needed? Fix: #5779 no ### How was this patch tested? run SQL with access Aliyun OSS data
apache · Jan 7, 2025 · 79536fe · 79536fe
1 parent 8e48de3
commit 79536fe
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 1 deletion.
diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
@@ -38,13 +38,15 @@ cd ${gravitino_home}
 ./gradlew :bundles:gcp-bundle:jar
 ./gradlew :bundles:aws-bundle:jar
 ./gradlew :bundles:azure-bundle:jar
+./gradlew :bundles:aliyun-bundle:jar
 
 # prepare bundle jar
 cd ${iceberg_rest_server_dir}
 mkdir -p bundles
 cp ${gravitino_home}/bundles/gcp-bundle/build/libs/gravitino-gcp-bundle-*.jar bundles/
 cp ${gravitino_home}/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*.jar bundles/
 cp ${gravitino_home}/bundles/azure-bundle/build/libs/gravitino-azure-bundle-*.jar bundles/
+cp ${gravitino_home}/bundles/aliyun-bundle/build/libs/gravitino-aliyun-bundle-*.jar bundles/
 
 iceberg_gcp_bundle="iceberg-gcp-bundle-1.5.2.jar"
 if [ ! -f "bundles/${iceberg_gcp_bundle}" ]; then
@@ -61,6 +63,11 @@ if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then
   curl -L -s -o bundles/${iceberg_azure_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/1.5.2/${iceberg_azure_bundle}
 fi
 
+iceberg_aliyun_bundle="iceberg-aliyun-bundle-1.5.2.jar"
+if [ ! -f "bundles/${iceberg_aliyun_bundle}" ]; then
+  curl -L -s -o bundles/${iceberg_aliyun_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aliyun-bundle/1.5.2/${iceberg_aliyun_bundle}
+fi
+
 # download jdbc driver
 curl -L -s -o bundles/sqlite-jdbc-3.42.0.0.jar https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar
 

diff --git a/dev/docker/iceberg-rest-server/rewrite_config.py b/dev/docker/iceberg-rest-server/rewrite_config.py
@@ -36,6 +36,13 @@
   "GRAVITINO_AZURE_TENANT_ID" : "azure-tenant-id",
   "GRAVITINO_AZURE_CLIENT_ID" : "azure-client-id",
   "GRAVITINO_AZURE_CLIENT_SECRET" : "azure-client-secret",
+  "GRAVITINO_OSS_ACCESS_KEY": "oss-access-key-id",
+  "GRAVITINO_OSS_SECRET_KEY": "oss-secret-access-key",
+  "GRAVITINO_OSS_ENDPOINT": "oss-endpoint",
+  "GRAVITINO_OSS_REGION": "oss-region",
+  "GRAVITINO_OSS_ROLE_ARN": "oss-role-arn",
+  "GRAVITINO_OSS_EXTERNAL_ID": "oss-external-id",
+
 }
 
 init_config = {

diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md
@@ -59,6 +59,8 @@ docker run --rm -d -p 9001:9001 apache/gravitino-iceberg-rest:0.7.0-incubating
 ```
 
 Changelog
+- apache/gravitino-iceberg-rest:0.8.0-incubating
+  - Supports OSS and ADLS storage.
 
 - apache/gravitino-iceberg-rest:0.7.0-incubating
   - Using JDBC catalog backend.

diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
@@ -441,7 +441,7 @@ SELECT * FROM dml.test;
 You could run Gravitino Iceberg REST server though docker container:
 
 ```shell
-docker run -d -p 9001:9001 apache/gravitino-iceberg-rest:0.7.0-incubating
+docker run -d -p 9001:9001 apache/gravitino-iceberg-rest:0.8.0-incubating
 ```
 
 Gravitino Iceberg REST server in docker image could access local storage by default, you could set the following environment variables if the storage is cloud/remote storage like S3, please refer to [storage section](#storage) for more details.
@@ -464,6 +464,12 @@ Gravitino Iceberg REST server in docker image could access local storage by defa
 | `GRAVITINO_AZURE_TENANT_ID`            | `gravitino.iceberg-rest.azure-tenant-id`            | 0.8.0-incubating |
 | `GRAVITINO_AZURE_CLIENT_ID`            | `gravitino.iceberg-rest.azure-client-id`            | 0.8.0-incubating |
 | `GRAVITINO_AZURE_CLIENT_SECRET`        | `gravitino.iceberg-rest.azure-client-secret`        | 0.8.0-incubating |
+| `GRAVITINO_OSS_ACCESS_KEY`             | `gravitino.iceberg-rest.oss-access-key-id`          | 0.8.0-incubating |
+| `GRAVITINO_OSS_SECRET_KEY`             | `gravitino.iceberg-rest.oss-secret-access-key`      | 0.8.0-incubating |
+| `GRAVITINO_OSS_ENDPOINT`               | `gravitino.iceberg-rest.oss-endpoint`               | 0.8.0-incubating |
+| `GRAVITINO_OSS_REGION`                 | `gravitino.iceberg-rest.oss-region`                 | 0.8.0-incubating |
+| `GRAVITINO_OSS_ROLE_ARN`               | `gravitino.iceberg-rest.oss-role-arn`               | 0.8.0-incubating |
+| `GRAVITINO_OSS_EXTERNAL_ID`            | `gravitino.iceberg-rest.oss-external-id`            | 0.8.0-incubating |
 
 The below environment is deprecated, please use the corresponding configuration items instead.