From e7eea4f2f923eb906af191460cc30755fe674120 Mon Sep 17 00:00:00 2001 From: David Byron <82477955+dbyron-sf@users.noreply.github.com> Date: Wed, 1 May 2024 21:24:59 -0700 Subject: [PATCH] feat(build): add fiat-integration module to exercise the just-built docker imageTest docker image (#6206) * fix(web): replace deprecated spring.profiles in configuration with spring.config.activate.on-profile to remove these warnings: 2024-05-01 21:29:23.746 WARN 1 --- [ main] o.s.b.c.config.ConfigDataEnvironment : Property 'spring.profiles' imported from location 'class path resource [clouddriver.yml]' is invalid and should be replaced with 'spring.config.activate.on-profile' [origin: class path resource [clouddriver.yml] - 375:13] 2024-05-01 21:29:23.746 WARN 1 --- [ main] o.s.b.c.config.ConfigDataEnvironment : Property 'spring.profiles' imported from location 'class path resource [clouddriver.yml]' is invalid and should be replaced with 'spring.config.activate.on-profile' [origin: class path resource [clouddriver.yml] - 363:13] 2024-05-01 21:29:23.746 WARN 1 --- [ main] o.s.b.c.config.ConfigDataEnvironment : Property 'spring.profiles' imported from location 'class path resource [clouddriver.yml]' is invalid and should be replaced with 'spring.config.activate.on-profile' [origin: class path resource [clouddriver.yml] - 350:13] 2024-05-01 21:29:23.746 WARN 1 --- [ main] o.s.b.c.config.ConfigDataEnvironment : Property 'spring.profiles' imported from location 'class path resource [clouddriver.yml]' is invalid and should be replaced with 'spring.config.activate.on-profile' [origin: class path resource [clouddriver.yml] - 312:13] See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-Config-Data-Migration-Guide#profile-specific-documents. * feat(docker): add HEALTHCHECK to facilitate testing container startup * feat(build): add clouddriver-integration module to exercise the just-built docker image * feat(gha): run integration test in pr builds multi-arch with --load doesn't work, so add a separate step using the local platform to make an image available for testing. see docker/buildx#59 * feat(gha): run integration test in branch builds * fix(docker): reduce the chance for false positives in the health check In case the health check contains more detailed information where one check could report UP but the overall status is down/out of service/etc. See https://docs.spring.io/spring-boot/docs/2.6.15/reference/html/actuator.html#actuator.endpoints.health for more. (cherry picked from commit 9ea2224ef03033d99535678a4aa176a3f574574f) --- .github/workflows/build.yml | 13 ++ .github/workflows/pr.yml | 13 ++ Dockerfile.java11.slim | 3 +- Dockerfile.java11.ubuntu | 2 + Dockerfile.slim | 3 +- Dockerfile.ubuntu | 2 + .../clouddriver-integration.gradle | 24 ++++ .../clouddriver/StandaloneContainerTest.java | 133 ++++++++++++++++++ .../src/test/resources/logback.xml | 36 +++++ clouddriver-web/config/clouddriver.yml | 16 ++- settings.gradle | 1 + 11 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 clouddriver-integration/clouddriver-integration.gradle create mode 100644 clouddriver-integration/src/test/java/com/netflix/spinnaker/clouddriver/StandaloneContainerTest.java create mode 100644 clouddriver-integration/src/test/resources/logback.xml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2cc14b4fc48..9e4505f77a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,6 +41,19 @@ jobs: env: ORG_GRADLE_PROJECT_version: ${{ steps.build_variables.outputs.VERSION }} run: ./gradlew build --stacktrace ${{ steps.build_variables.outputs.REPO }}-web:installDist + - name: Build local slim container image for testing + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile.slim + load: true + platforms: local + tags: | + "${{ steps.build_variables.outputs.REPO }}:${{ steps.build_variables.outputs.VERSION }}-unvalidated" + - name: Test local slim container image + env: + FULL_DOCKER_IMAGE_NAME: "${{ steps.build_variables.outputs.REPO }}:${{ steps.build_variables.outputs.VERSION }}-unvalidated" + run: ./gradlew ${{ steps.build_variables.outputs.REPO }}-integration:test - name: Login to GAR # Only run this on repositories in the 'spinnaker' org, not on forks. if: startsWith(github.repository, 'spinnaker/') diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a4685ea4bc1..bfc6dc13084 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -75,3 +75,16 @@ jobs: tags: | "${{ env.CONTAINER_REGISTRY }}/${{ steps.build_variables.outputs.REPO }}:latest-java11-ubuntu" "${{ env.CONTAINER_REGISTRY }}/${{ steps.build_variables.outputs.REPO }}:${{ steps.build_variables.outputs.VERSION }}-java11-ubuntu" + - name: Build local slim container image for testing + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile.slim + load: true + platforms: local + tags: | + "${{ steps.build_variables.outputs.REPO }}:${{ steps.build_variables.outputs.VERSION }}" + - name: Test local slim container image + env: + FULL_DOCKER_IMAGE_NAME: "${{ steps.build_variables.outputs.REPO }}:${{ steps.build_variables.outputs.VERSION }}" + run: ./gradlew ${{ steps.build_variables.outputs.REPO }}-integration:test diff --git a/Dockerfile.java11.slim b/Dockerfile.java11.slim index 52ce2c6a8b6..816ea0e115e 100644 --- a/Dockerfile.java11.slim +++ b/Dockerfile.java11.slim @@ -17,6 +17,7 @@ RUN apk update \ && apk --no-cache add --update \ bash \ ca-certificates \ + curl \ wget \ openjdk11 \ git \ @@ -55,5 +56,5 @@ COPY clouddriver-web/build/install/clouddriver /opt/clouddriver RUN mkdir -p /opt/clouddriver/plugins && chown -R spinnaker:nogroup /opt/clouddriver/plugins USER spinnaker - +HEALTHCHECK CMD curl --fail http://localhost:7002/health CMD ["/opt/clouddriver/bin/clouddriver"] diff --git a/Dockerfile.java11.ubuntu b/Dockerfile.java11.ubuntu index a5315b33580..5b2934b8fe7 100644 --- a/Dockerfile.java11.ubuntu +++ b/Dockerfile.java11.ubuntu @@ -14,6 +14,7 @@ RUN apt-get update && apt-get install -y curl gnupg && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y \ + curl \ openjdk-11-jre-headless \ wget \ python3-pip \ @@ -51,4 +52,5 @@ RUN adduser --system --uid 10111 --group spinnaker COPY clouddriver-web/build/install/clouddriver /opt/clouddriver RUN mkdir -p /opt/clouddriver/plugins && chown -R spinnaker:nogroup /opt/clouddriver/plugins USER spinnaker +HEALTHCHECK CMD curl --fail http://localhost:7002/health CMD ["/opt/clouddriver/bin/clouddriver"] diff --git a/Dockerfile.slim b/Dockerfile.slim index 2be397c24b1..ecbfc417daa 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -17,6 +17,7 @@ RUN apk update \ && apk --no-cache add --update \ bash \ ca-certificates \ + curl \ wget \ openjdk17 \ git \ @@ -55,5 +56,5 @@ COPY clouddriver-web/build/install/clouddriver /opt/clouddriver RUN mkdir -p /opt/clouddriver/plugins && chown -R spinnaker:nogroup /opt/clouddriver/plugins USER spinnaker - +HEALTHCHECK CMD curl --fail http://localhost:7002/health CMD ["/opt/clouddriver/bin/clouddriver"] diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 8f57287f548..cde1e810900 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -14,6 +14,7 @@ RUN apt-get update && apt-get install -y curl gnupg && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y \ + curl \ openjdk-17-jre-headless \ wget \ python3-pip \ @@ -51,4 +52,5 @@ RUN adduser --system --uid 10111 --group spinnaker COPY clouddriver-web/build/install/clouddriver /opt/clouddriver RUN mkdir -p /opt/clouddriver/plugins && chown -R spinnaker:nogroup /opt/clouddriver/plugins USER spinnaker +HEALTHCHECK CMD curl --fail http://localhost:7002/health CMD ["/opt/clouddriver/bin/clouddriver"] diff --git a/clouddriver-integration/clouddriver-integration.gradle b/clouddriver-integration/clouddriver-integration.gradle new file mode 100644 index 00000000000..d19d36f3919 --- /dev/null +++ b/clouddriver-integration/clouddriver-integration.gradle @@ -0,0 +1,24 @@ +dependencies { + testImplementation "com.fasterxml.jackson.core:jackson-databind" + testImplementation "org.assertj:assertj-core" + testImplementation "org.junit.jupiter:junit-jupiter-api" + testImplementation "org.slf4j:slf4j-api" + testImplementation "org.testcontainers:testcontainers" + testImplementation "org.testcontainers:junit-jupiter" + testRuntimeOnly "ch.qos.logback:logback-classic" +} + +test.configure { + def fullDockerImageName = System.getenv('FULL_DOCKER_IMAGE_NAME') + onlyIf("there is a docker image to test") { + fullDockerImageName != null && fullDockerImageName.trim() != '' + } +} + +test { + // So stdout and stderr from the just-built container are available in CI + testLogging.showStandardStreams = true + + // Run the tests when the docker image changes + inputs.property 'fullDockerImageName', System.getenv('FULL_DOCKER_IMAGE_NAME') +} diff --git a/clouddriver-integration/src/test/java/com/netflix/spinnaker/clouddriver/StandaloneContainerTest.java b/clouddriver-integration/src/test/java/com/netflix/spinnaker/clouddriver/StandaloneContainerTest.java new file mode 100644 index 00000000000..59fe02b29d8 --- /dev/null +++ b/clouddriver-integration/src/test/java/com/netflix/spinnaker/clouddriver/StandaloneContainerTest.java @@ -0,0 +1,133 @@ +/* + * Copyright 2024 Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +@Testcontainers +class StandaloneContainerTest { + + private static final String REDIS_NETWORK_ALIAS = "redisHost"; + + private static final int REDIS_PORT = 6379; + + private static final Logger logger = LoggerFactory.getLogger(StandaloneContainerTest.class); + + private static final Network network = Network.newNetwork(); + + private static final GenericContainer redis = + new GenericContainer(DockerImageName.parse("library/redis:5-alpine")) + .withNetwork(network) + .withNetworkAliases(REDIS_NETWORK_ALIAS) + .withExposedPorts(REDIS_PORT); + + private static GenericContainer clouddriverContainer; + + @BeforeAll + static void setupOnce() throws Exception { + String fullDockerImageName = System.getenv("FULL_DOCKER_IMAGE_NAME"); + + // Skip the tests if there's no docker image. This allows gradlew build to work. + assumeTrue(fullDockerImageName != null); + + redis.start(); + + DockerImageName dockerImageName = DockerImageName.parse(fullDockerImageName); + + clouddriverContainer = + new GenericContainer(dockerImageName) + .withNetwork(network) + .withExposedPorts(7002) + .dependsOn(redis) + .waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(90))) + .withEnv("SPRING_APPLICATION_JSON", getSpringApplicationJson()); + + Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(logger); + clouddriverContainer.start(); + clouddriverContainer.followOutput(logConsumer); + } + + private static String getSpringApplicationJson() throws JsonProcessingException { + String redisUrl = "redis://" + REDIS_NETWORK_ALIAS + ":" + REDIS_PORT; + logger.info("redisUrl: '{}'", redisUrl); + Map properties = + Map.of("redis.connection", redisUrl, "services.fiat.baseUrl", "http://nowhere"); + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(properties); + } + + @AfterAll + static void cleanupOnce() { + if (clouddriverContainer != null) { + clouddriverContainer.stop(); + } + + if (redis != null) { + redis.stop(); + } + } + + @BeforeEach + void init(TestInfo testInfo) { + System.out.println("--------------- Test " + testInfo.getDisplayName()); + } + + @Test + void testHealthCheck() throws Exception { + // hit an arbitrary endpoint + HttpRequest request = + HttpRequest.newBuilder() + .uri( + new URI( + "http://" + + clouddriverContainer.getHost() + + ":" + + clouddriverContainer.getFirstMappedPort() + + "/health")) + .GET() + .build(); + + HttpClient client = HttpClient.newHttpClient(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + assertThat(response).isNotNull(); + logger.info("response: {}, {}", response.statusCode(), response.body()); + assertThat(response.statusCode()).isEqualTo(200); + } +} diff --git a/clouddriver-integration/src/test/resources/logback.xml b/clouddriver-integration/src/test/resources/logback.xml new file mode 100644 index 00000000000..6145d38780b --- /dev/null +++ b/clouddriver-integration/src/test/resources/logback.xml @@ -0,0 +1,36 @@ + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + diff --git a/clouddriver-web/config/clouddriver.yml b/clouddriver-web/config/clouddriver.yml index e5d802794f2..57407646d98 100644 --- a/clouddriver-web/config/clouddriver.yml +++ b/clouddriver-web/config/clouddriver.yml @@ -309,7 +309,9 @@ resilience4j.retry: --- spring: - profiles: prod + config: + activate: + on-profile: prod # example https configuration for client auth to services: #default: @@ -347,7 +349,9 @@ spring: # local profile is activated by default when running the application - override values here for local development # for production, set spring.profiles.active to select the appropriate profile for your environment spring: - profiles: local + config: + activate: + on-profile: local # an AWSCredentialsProvider that obtains session credentials via SSH through a bastion instance (useful for local development): #bastion: @@ -360,7 +364,9 @@ spring: # This profile is used in HA deployments for a clouddriver that handles read-only requests from # other services spring: - profiles: ro + config: + activate: + on-profile: ro redis: connection: ${services.redisRo.baseUrl:${services.redis.baseUrl}} @@ -372,7 +378,9 @@ caching: # This profile is used in HA deployments for a clouddriver that handles mutating requests from # other services, but does not run caching agents spring: - profiles: rw + config: + activate: + on-profile: rw caching: writeEnabled: false diff --git a/settings.gradle b/settings.gradle index 1a72c825135..7f7f9c6c524 100644 --- a/settings.gradle +++ b/settings.gradle @@ -65,6 +65,7 @@ include 'cats:cats-core', 'clouddriver-core-tck', 'clouddriver-elasticsearch', 'clouddriver-event', + 'clouddriver-integration', 'clouddriver-saga', 'clouddriver-saga-test', 'clouddriver-security',