diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml new file mode 100644 index 0000000000000..16470e1e4b283 --- /dev/null +++ b/.github/workflows/native.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: GraalVM Native Image Daily Continuous Integration + +on: + schedule: + - cron: '0 0 * * *' + +env: + HUB: ghcr.io/apache/shardingsphere + PROXY: ghcr.io/apache/shardingsphere-proxy-native + +jobs: + build: + if: github.repository == 'apache/shardingsphere' + timeout-minutes: 90 + permissions: + contents: read + packages: write + name: GraalVM Native Image on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + fail-fast: false + steps: + - uses: actions/checkout@v2 + - uses: graalvm/setup-graalvm@v1 + with: + version: '22.2.0' + java-version: '17' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} + cache: 'maven' + - name: Log in to the Container registry + uses: docker/login-action@v1.10.0 + with: + registry: ${{ env.HUB }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push docker image + run: ./mvnw -am -pl shardingsphere-distribution/shardingsphere-proxy-native-distribution -B -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat.skip=true -Pnative,docker.buildx.push.native -Dproxy.image.repository=${{ env.PROXY }} -Dproxy.image.tag=${{ github.sha }} clean package diff --git a/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md b/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md new file mode 100644 index 0000000000000..f5446d1e9ac9e --- /dev/null +++ b/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.cn.md @@ -0,0 +1,118 @@ ++++ +title = "构建 GraalVM Native Image(Alpha)" +weight = 2 ++++ + +## 背景信息 + +本节主要介绍如何通过 `GraalVM` 的 `native-image` 组件构建 ShardingSphere-Proxy 的 `Native Image` 和对应的 `Docker Image` +。 + +## 注意事项 + +- ShardingSphere Proxy 尚未准备好与 GraalVM Native Image 集成。 + 其在 https://github.com/apache/shardingsphere/actions/ 存在每日构建的任务用于测试构建。 + +- 若你发现构建过程存在缺失的 GraalVM Reachability Metadata, + 应当在 https://github.com/oracle/graalvm-reachability-metadata 打开新的 issue , + 并提交包含 ShardingSphere 自身或依赖的第三方库缺失的 GraalVM Reachability Metadata 的 PR。 + +- ShardingSphere 的 master 分支尚未准备好处理 Native Image 中的单元测试, + 需要等待 Junit 5 Platform 的集成,你总是需要在构建 GraalVM Native Image 的过程中, + 加上特定于 `GraalVM Native Build Tools` 的 `-DskipNativeTests` 或 `-DskipTests` 参数跳过 Native Image 中的单元测试。 + +- 本节假定处于 Linux(amd64,aarch64), MacOS( amd64 )或 Windows(amd64)环境。 + 如果你位于 MacOS(aarch64/M1) 环境, 你需要关注尚未关闭的 https://github.com/oracle/graal/issues/2666 。 + +## 前提条件 + +1. 根据 https://www.graalvm.org/downloads/ 要求安装和配置 JDK 17 对应的 `GraalVM CE` 或 `GraalVM EE`。 + 同时可以通过 `SDKMAN!` 安装 JDK 17 对应的 `GraalVM CE`。 + +2. 通过 `GraalVM Updater` 工具安装 `native-image` 组件。 + +3. 根据 https://www.graalvm.org/22.2/reference-manual/native-image/#prerequisites 的要求安装本地工具链。 + +4. 如果需要构建 Docker Image, 确保 `docker-cli` 在系统环境变量内。 + +## 操作步骤 + +1. 获取 Apache ShardingSphere Git Source + +- 在[下载页面](https://shardingsphere.apache.org/document/current/en/downloads/) + 或 https://github.com/apache/shardingsphere/tree/master 获取。 + +2. 在命令行构建产物, 分两种情形。 + +- 情形一:不需要使用存在 SPI 实现的 JAR 或第三方依赖的 JAR + +- 在 Git Source 同级目录下执行如下命令, 直接完成 Native Image 的构建。 + +```bash +./mvnw -am -pl shardingsphere-distribution/shardingsphere-proxy-native-distribution -B -Pnative -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat.skip=true clean package +``` + +- 情形二:需要使用存在 SPI 实现的 JAR 或 GPL V2 等 LICENSE 的第三方依赖的 JAR。 + +- 在 `shardingsphere-distribution/shardingsphere-proxy-native-distribution/pom.xml` 的 `dependencies` 加入存在 SPI 实现的 JAR + 或第三方依赖的 JAR。示例如下 + +```xml + + + + mysql + mysql-connector-java + 8.0.30 + + + org.apache.shardingsphere + shardingsphere-sql-translator-jooq-provider + 5.2.0 + + +``` + +- 通过命令行构建 GraalVM Native Image。 + +```bash +./mvnw -am -pl shardingsphere-distribution/shardingsphere-proxy-native-distribution -B -Pnative -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat.skip=true clean package +``` + +3. 通过命令行启动 Native Image, 需要带上两个参数, + 第一个参数为 ShardingSphere Proxy 使用的端口,第二个参数为你编写的包含 `server.yaml` 的 `/conf` 文件夹, + 假设已存在文件夹`./custom/conf`,示例为 + +```bash +./apache-shardingsphere-proxy 3307 ./custom/conf +``` + +4. 如果需要构建 Docker Image, 在添加后存在 SPI 实现的依赖或第三方依赖后, 在命令行执行如下命令。 + +```shell +./mvnw -am -pl shardingsphere-distribution/shardingsphere-proxy-native-distribution -B -Pnative,docker.native -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat.skip=true clean package +``` + +- 假设存在包含`server.yaml` 的 `conf` 文件夹为 `./custom/conf`,可通过如下的 `docker-compose.yml` 文件启动 GraalVM Native + Image 对应的 Docker Image。 + +```yaml +version: "3.8" + +services: + apache-shardingsphere-proxy-native: + image: apache/shardingsphere-proxy-native:latest + volumes: + - ./custom/conf:/conf + ports: + - "3307:3307" +``` + +- 如果您使用默认构建配置, 你当然可以为 `shardingsphere-distribution/shardingsphere-proxy-native-distribution/Dockerfile` + 使用 `scratch` 作为 base docker image。 + 但如果您主动为`pom.xml`的`native profile`添加`jvmArgs`为`-H:+StaticExecutableWithDynamicLibC`, + 以静态链接除 `glic` 之外的所有内容,您应该切换 base image 到 `busybox:glic`。 + 参考 https://www.graalvm.org/22.2/reference-manual/native-image/guides/build-static-executables/ 。 + 另请注意,某些第三方依赖将需要更多系统库,例如 `libdl`。 + 因此请确保根据您的使用情况调整 base docker image 和`shardingsphere-distribution/shardingsphere-proxy-native-distribution` + 下的 `pom.xml` 和 `Dockerfile` 的内容。 diff --git a/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md b/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md new file mode 100644 index 0000000000000..b253007729b3d --- /dev/null +++ b/docs/document/content/user-manual/shardingsphere-proxy/startup/graalvm-native-image.en.md @@ -0,0 +1,125 @@ ++++ +title = "Build GraalVM Native Image(Alpha)" +weight = 2 ++++ + +## Background + +This section mainly introduces how to build the `Native Image` of ShardingSphere-Proxy and the +corresponding `Docker Image` through the `native-image` component of `GraalVM`. + +## Notice + +- ShardingSphere Proxy is not yet ready to integrate with GraalVM Native Image. + It has daily build tasks at https://github.com/apache/shardingsphere/actions/ for testing builds. + +- If you find that the build process has missing GraalVM Reachability Metadata, + A new issue should be opened at https://github.com/oracle/graalvm-reachability-metadata, + And submit a PR containing GraalVM Reachability Metadata missing from ShardingSphere itself or dependent third-party + libraries. + +- The master branch of ShardingSphere is not yet ready to handle unit tests in Native Image, + Need to wait for the integration of Junit 5 Platform, you always need to build GraalVM Native Image in the process, + Plus `-DskipNativeTests` or `-DskipTests` parameter specific to `GraalVM Native Build Tools` to skip unit tests in + Native Image. + +- This section assumes a Linux (amd64, aarch64), MacOS (amd64) or Windows (amd64) environment. + If you are on MacOS(aarch64/M1) environment, you need to follow https://github.com/oracle/graal/issues/2666 which is + not closed yet. + +## Premise + +1. Install and configure `GraalVM CE` or `GraalVM EE` for JDK 17 according to https://www.graalvm.org/downloads/. + `GraalVM CE` for JDK 17 can also be installed via `SDKMAN!`. + +2. Install the `native-image` component via the `GraalVM Updater` tool. + +3. Install the local toolchain as required by https://www.graalvm.org/22.2/reference-manual/native-image/#prerequisites. + +4. If you need to build a Docker Image, make sure `docker-cli` is in the system environment variables. + +## Steps + +1. Get Apache ShardingSphere Git Source + +- Get it at the [download page](https://shardingsphere.apache.org/document/current/en/downloads/) + or https://github.com/apache/shardingsphere/tree/master. + +2. Build the product on the command line, in two cases. + +- Scenario 1: No need to use JARs with SPI implementations or 3rd party dependencies + +- Execute the following command in the same directory of Git Source to directly complete the construction of Native + Image. + +```bash +./mvnw -am -pl shardingsphere-distribution/shardingsphere-proxy-native-distribution -B -Pnative -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat.skip=true clean package +``` + +- Scenario 2: It is necessary to use a JAR that has an SPI implementation or a third-party dependent JAR of a LICENSE + such as GPL V2. + +- Add SPI implementation JARs or third-party dependent JARs to `dependencies` + in `shardingsphere-distribution/shardingsphere-proxy-native-distribution/pom.xml`. Examples are as follows + +```xml + + + + mysql + mysql-connector-java + 8.0.30 + + + org.apache.shardingsphere + shardingsphere-sql-translator-jooq-provider + 5.2.0 + + +``` + +- Build GraalVM Native Image via command line. + +```bash +./mvnw -am -pl shardingsphere-distribution/shardingsphere-proxy-native-distribution -B -Pnative -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat.skip=true clean package +``` + +3. Start Native Image through the command line, you need to bring two parameters, + The first parameter is the port used by ShardingSphere Proxy, and the second parameter is the `/conf` folder that + contains `server.yaml` written by you, + Assuming the folder `./custom/conf` already exists, the example is + +```bash +./apache-shardingsphere-proxy 3307 ./custom/conf +```` + +4. If you need to build a Docker Image, after adding the dependencies of the SPI implementation or third-party + dependencies, execute the following commands on the command line. + +```shell +./mvnw -am -pl shardingsphere-distribution/shardingsphere-proxy-native-distribution -B -Pnative,docker.native -DskipTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Drat .skip=true clean package +``` + +- Assuming that there is a `conf` folder containing `server.yaml` as `./custom/conf`, you can start the Docker Image + corresponding to GraalVM Native Image through the following `docker-compose.yml` file. + +```yaml +version: "3.8" + +services: + apache-shardingsphere-proxy-native: + image: apache/shardingsphere-proxy-native:latest + volumes: + - ./custom/conf:/conf + ports: + - "3307:3307" +``` + +- If you use the default build configuration, you can of course use `scratch` as the base docker image + for `shardingsphere-distribution/shardingsphere-proxy-native-distribution/Dockerfile`. + But if you actively add `jvmArgs` to `-H:+StaticExecutableWithDynamicLibC` for the `native profile` of `pom.xml`, + To statically link everything except `glic`, you should switch the base image to `busybox:glic`. Refer + to https://www.graalvm.org/22.2/reference-manual/native-image/guides/build-static-executables/. + Also note that some third-party dependencies will require more system libraries, such as `libdl`. + So make sure to adjust the base docker image and the content of `pom.xml` and `Dockerfile` + under `shardingsphere-distribution/shardingsphere-proxy-native-distribution` according to your usage. diff --git a/shardingsphere-distribution/pom.xml b/shardingsphere-distribution/pom.xml index c5cdf4aaccb09..32c55ea9ec68f 100644 --- a/shardingsphere-distribution/pom.xml +++ b/shardingsphere-distribution/pom.xml @@ -32,6 +32,7 @@ shardingsphere-src-distribution shardingsphere-jdbc-distribution shardingsphere-proxy-distribution + shardingsphere-proxy-native-distribution diff --git a/shardingsphere-distribution/shardingsphere-proxy-native-distribution/Dockerfile b/shardingsphere-distribution/shardingsphere-proxy-native-distribution/Dockerfile new file mode 100644 index 0000000000000..3a188c7a28e16 --- /dev/null +++ b/shardingsphere-distribution/shardingsphere-proxy-native-distribution/Dockerfile @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM busybox AS prepare + +RUN mkdir -p /conf/ + +FROM scratch + +MAINTAINER ShardingSphere "dev@shardingsphere.apache.org" + +COPY --from=prepare /conf/ /conf + +ARG APP_NAME + +ADD target/${APP_NAME} ./ + +ENTRYPOINT ./${APP_NAME} 3307 /conf diff --git a/shardingsphere-distribution/shardingsphere-proxy-native-distribution/pom.xml b/shardingsphere-distribution/shardingsphere-proxy-native-distribution/pom.xml new file mode 100644 index 0000000000000..d2f83befcfca4 --- /dev/null +++ b/shardingsphere-distribution/shardingsphere-proxy-native-distribution/pom.xml @@ -0,0 +1,260 @@ + + + + + + shardingsphere-distribution + org.apache.shardingsphere + 5.2.1-SNAPSHOT + + 4.0.0 + + shardingsphere-proxy-native-distribution + + + apache-shardingsphere-proxy-native + apache/shardingsphere-proxy-native + 3.1.0 + 0.9.14 + + + + + org.apache.shardingsphere + shardingsphere-proxy-bootstrap + ${project.version} + + + + com.zaxxer + HikariCP + compile + + + + org.postgresql + postgresql + runtime + + + + org.opengauss + opengauss-jdbc + runtime + + + + ch.qos.logback + logback-classic + runtime + + + + + + native + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + 17 + 17 + --enable-preview + + + + org.graalvm.buildtools + native-maven-plugin + ${native.maven.plugin.version} + true + + + build-native + + compile-no-fork + + package + + + test-native + + test + + test + + + + ${native.image.name} + org.apache.shardingsphere.proxy.Bootstrap + false + true + + --report-unsupported-elements-at-runtime + + + --enable-preview + + + true + + + + + + + + docker.native + + ${native.image.repository} + ${project.version} + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + build + + exec + + package + + docker + + build + --pull + --build-arg + APP_NAME=${native.image.name} + . + + + + + + + + + + docker.buildx.push.native + + linux/amd64,linux/arm64 + ${native.image.repository} + ${project.version} + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + create builder + + exec + + package + + docker + + buildx + create + --use + --driver + docker-container + --name + shardingsphere-builder + --platform + ${proxy.image.platform} + + + + + build + + exec + + package + + docker + + buildx + build + --pull + --platform + ${proxy.image.platform} + --build-arg + APP_NAME=${native.image.name} + . + + + + + push + + exec + + package + + docker + + buildx + build + --push + --platform + ${proxy.image.platform} + --build-arg + APP_NAME=${native.image.name} + . + -t + ${proxy.image.repository}:${proxy.image.tag} + -t + ${proxy.image.repository}:latest + + + + + cleanup builder + + exec + + package + + docker + + buildx + rm + shardingsphere-builder + + + + + + + + + +