Skip to content

Commit

Permalink
Enable Java SDK Distroless container image variants. (#33173)
Browse files Browse the repository at this point in the history
* Enable Java SDK Distroless container image variant

* Add LANG environment and /usr/lib/locale

* Use examples tests instead
  • Loading branch information
damondouglas authored Nov 23, 2024
1 parent d342dd3 commit 09c1c9e
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 2 deletions.
5 changes: 4 additions & 1 deletion .github/trigger_files/beam_PostCommit_Java.json
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
{}
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
}
2 changes: 1 addition & 1 deletion .github/trigger_files/beam_PostCommit_Java_DataflowV2.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
"modification": 2
}
63 changes: 63 additions & 0 deletions runners/google-cloud-dataflow-java/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,69 @@ def createRunnerV2ValidatesRunnerTest = { Map args ->
}
}

tasks.register('examplesJavaRunnerV2IntegrationTestDistroless', Test.class) {
group = "verification"
dependsOn 'buildAndPushDistrolessContainerImage'
def javaVer = project.findProperty('testJavaVersion')
def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
def tag = project.findProperty('dockerTag')
def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}"
def pipelineOptions = [
"--runner=TestDataflowRunner",
"--project=${gcpProject}",
"--region=${gcpRegion}",
"--tempRoot=${dataflowValidatesTempRoot}",
"--sdkContainerImage=${imageURL}",
"--experiments=use_unified_worker,use_runner_v2",
"--firestoreDb=${firestoreDb}",
]
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions)

include '**/*IT.class'

maxParallelForks 4
classpath = configurations.examplesJavaIntegrationTest
testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs)
useJUnit { }
}

tasks.register('buildAndPushDistrolessContainerImage', Task.class) {
// Only Java 17 and 21 are supported.
// See https://github.com/GoogleContainerTools/distroless/tree/main/java#image-contents.
def allowed = ["java17", "java21"]
doLast {
def javaVer = project.findProperty('testJavaVersion')
if (!allowed.contains(javaVer)) {
throw new GradleException("testJavaVersion must be one of ${allowed}, got: ${javaVer}")
}
if (!project.hasProperty('dockerTag')) {
throw new GradleException("dockerTag is missing but required")
}
def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
def tag = project.findProperty('dockerTag')
def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}"
exec {
executable 'docker'
workingDir rootDir
args = [
'buildx',
'build',
'-t',
imageURL,
'-f',
'sdks/java/container/Dockerfile-distroless',
"--build-arg=BEAM_BASE=gcr.io/apache-beam-testing/beam-sdk/beam_${javaVer}_sdk",
"--build-arg=DISTROLESS_BASE=gcr.io/distroless/${javaVer}-debian12",
'.'
]
}
exec {
executable 'docker'
args = ['push', imageURL]
}
}
}

// Push docker images to a container registry for use within tests.
// NB: Tasks which consume docker images from the registry should depend on this
// task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct
Expand Down
42 changes: 42 additions & 0 deletions sdks/java/container/Dockerfile-distroless
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

# ARG BEAM_BASE is the Beam SDK container image built using sdks/python/container/Dockerfile.
ARG BEAM_BASE

# ARG DISTROLESS_BASE is the distroless container image URL. For available distroless Java images,
# see https://github.com/GoogleContainerTools/distroless/tree/main?tab=readme-ov-file#what-images-are-available.
# Only Java versions 17 and 21 are supported.
ARG DISTROLESS_BASE
FROM ${BEAM_BASE} AS base
ARG TARGETARCH
ENV LANG C.UTF-8

LABEL Author="Apache Beam <[email protected]>"

RUN if [ -z "${TARGETARCH}" ]; then echo "fatal: TARGETARCH not set; run as docker buildx build or use --build-arg=TARGETARCH=amd64|arm64" >&2; exit 1; fi

FROM ${DISTROLESS_BASE}:latest-${TARGETARCH} AS distroless

COPY --from=base /opt /opt

# Along with the LANG environment variable above, prevents internally discovered failing bugs related to Dataflow Flex
# template character encodings.
COPY --from=base /usr/lib/locale /usr/lib/locale

ENTRYPOINT ["/opt/apache/beam/boot"]

0 comments on commit 09c1c9e

Please sign in to comment.