Skip to content

Commit

Permalink
Fix K8s/OpenShift deployment when using management interface+https
Browse files Browse the repository at this point in the history
Note that these changes will move the management ssl configuration to built-time configuration. 

Moreover, the logic to select the HTTPS schema mimics to the logic in VertxHttpRecorder.initializeMainHttpServer method.

Fix quarkusio#32225
  • Loading branch information
Sgitario committed May 15, 2023
1 parent e03a978 commit 222933c
Show file tree
Hide file tree
Showing 14 changed files with 298 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,22 @@
public final class KubernetesProbePortNameBuildItem extends SimpleBuildItem {

private final String name;
private final String schema;

public KubernetesProbePortNameBuildItem(String name) {
this(name, null);
}

public KubernetesProbePortNameBuildItem(String name, String schema) {
this.name = name;
this.schema = schema;
}

public String getName() {
return name;
}

public String getSchema() {
return schema;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -133,15 +133,18 @@ public static List<DecoratorBuildItem> createDecorators(String clusterKind,
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, clusterKind, LIVENESS_PROBE, config.livenessProbe,
portName,
ports));
ports,
config.ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, clusterKind, READINESS_PROBE, config.readinessProbe,
portName,
ports));
ports,
config.ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, clusterKind, STARTUP_PROBE, config.startupProbe,
portName,
ports));
ports,
config.ports));

// Handle init Containers
result.addAll(KubernetesCommonHelper.createInitContainerDecorators(clusterKind, name, initContainers, result));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ public class KubernetesCommonHelper {
"Deployment", "DeploymentConfig" };
private static final String DEFAULT_ROLE_NAME_VIEW = "view";
private static final List<String> LIST_WITH_EMPTY = List.of("");
private static final String SCHEMA_HTTP = "HTTP";
private static final String SCHEMA_HTTPS = "HTTPS";

public static Optional<Project> createProject(ApplicationInfoBuildItem app,
Optional<CustomProjectRootBuildItem> customProjectRoot, OutputTargetBuildItem outputTarget,
Expand Down Expand Up @@ -946,7 +948,8 @@ private static List<DecoratorBuildItem> createAnnotationDecorators(Optional<Proj
public static DecoratorBuildItem createProbeHttpPortDecorator(String name, String target, String probeKind,
ProbeConfig probeConfig,
Optional<KubernetesProbePortNameBuildItem> portName,
List<KubernetesPortBuildItem> ports) {
List<KubernetesPortBuildItem> ports,
Map<String, PortConfig> portsFromConfig) {

//1. check if `httpActionPort` is defined
//2. lookup port by `httpPortName`
Expand All @@ -958,7 +961,26 @@ public static DecoratorBuildItem createProbeHttpPortDecorator(String name, Strin
Integer port = probeConfig.httpActionPort
.orElse(ports.stream().filter(p -> httpPortName.equals(p.getName()))
.map(KubernetesPortBuildItem::getPort).findFirst().orElse(DEFAULT_HTTP_PORT));
return new DecoratorBuildItem(target, new ApplyHttpGetActionPortDecorator(name, name, port, probeKind));

// Resolve schema property from:
String schema;
if (probeConfig.httpActionSchema.isPresent()) {
// 1. User in Probe config
schema = probeConfig.httpActionSchema.get();
} else if (portsFromConfig.containsKey(httpPortName) && portsFromConfig.get(httpPortName).tls) {
// 2. User in Ports config
schema = SCHEMA_HTTPS;
} else if (portName.isPresent()
&& portName.get().getSchema() != null
&& portName.get().getName().equals(httpPortName)) {
// 3. Extensions
schema = portName.get().getSchema();
} else {
// 4. Using the port number.
schema = port != null && (port == 443 || port == 8443) ? SCHEMA_HTTPS : SCHEMA_HTTP;
}

return new DecoratorBuildItem(target, new ApplyHttpGetActionPortDecorator(name, name, port, probeKind, schema));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -322,15 +322,16 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
// Probe port handling
result.add(KubernetesCommonHelper.createProbeHttpPortDecorator(name, OPENSHIFT, LIVENESS_PROBE, config.livenessProbe,
portName,
ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, OPENSHIFT, READINESS_PROBE, config.readinessProbe,
portName,
ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, OPENSHIFT, STARTUP_PROBE, config.startupProbe,
portName,
ports));
ports,
config.ports));
result.add(KubernetesCommonHelper.createProbeHttpPortDecorator(name, OPENSHIFT, READINESS_PROBE, config.readinessProbe,
portName,
ports,
config.ports));
result.add(KubernetesCommonHelper.createProbeHttpPortDecorator(name, OPENSHIFT, STARTUP_PROBE, config.startupProbe,
portName,
ports,
config.ports));

// Handle non-openshift builds
if (deploymentKind == DeploymentResourceKind.DeploymentConfig
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,10 @@ public class PortConfig {
*/
public OptionalInt nodePort;

/**
* If enabled, the port will be configured to use the schema HTTPS.
*/
@ConfigItem(defaultValue = "false")
public boolean tls;

}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ public class ProbeConfig {
@ConfigItem
Optional<String> httpActionPath;

/**
* The schema of the {@literal HTTP get} action. Can be either "HTTP" or "HTTPS".
*/
@ConfigItem
Optional<String> httpActionSchema;

/**
* The command to use for the probe.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,15 +257,18 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, KUBERNETES, LIVENESS_PROBE, config.livenessProbe,
portName,
ports));
ports,
config.ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, KUBERNETES, READINESS_PROBE, config.readinessProbe,
portName,
ports));
ports,
config.ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, KUBERNETES, STARTUP_PROBE, config.startupProbe,
portName,
ports));
ports,
config.ports));

// Handle remote debug configuration
if (config.remoteDebug.enabled) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.BooleanSupplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.eclipse.microprofile.config.Config;
import org.eclipse.microprofile.config.ConfigProvider;
import org.eclipse.microprofile.health.Liveness;
import org.eclipse.microprofile.health.Readiness;
Expand Down Expand Up @@ -102,6 +104,15 @@ class SmallRyeHealthProcessor {
private static final String BRANDING_FAVICON_GENERAL = BRANDING_DIR + "favicon.ico";
private static final String BRANDING_FAVICON_MODULE = BRANDING_DIR + "smallrye-health-ui.ico";

// For Kubernetes exposing
private static final String SCHEMA_HTTP = "HTTP";
private static final String SCHEMA_HTTPS = "HTTPS";

// For Management ports
private static final String MANAGEMENT_SSL_PREFIX = "quarkus.management.ssl.certificate.";
private static final List<String> MANAGEMENT_SSL_PROPERTIES = List.of("key-store-file", "trust-store-file", "files",
"key-files");

static class OpenAPIIncluded implements BooleanSupplier {
HealthBuildTimeConfig config;

Expand Down Expand Up @@ -344,7 +355,7 @@ public void kubernetes(NonApplicationRootPathBuildItem nonApplicationRootPathBui

if (managementInterfaceBuildTimeConfig.enabled) {
// Switch to the "management" port
port.produce(new KubernetesProbePortNameBuildItem("management"));
port.produce(new KubernetesProbePortNameBuildItem("management", selectSchemaForManagement()));
}

livenessPathItemProducer.produce(
Expand Down Expand Up @@ -466,4 +477,17 @@ public String updateApiUrl(String original, String healthPath) {
private static boolean shouldInclude(LaunchModeBuildItem launchMode, SmallRyeHealthConfig healthConfig) {
return launchMode.getLaunchMode().isDevOrTest() || healthConfig.ui.alwaysInclude;
}

private static String selectSchemaForManagement() {
Config config = ConfigProvider.getConfig();
for (String sslProperty : MANAGEMENT_SSL_PROPERTIES) {
Optional<List<String>> property = config.getOptionalValues(MANAGEMENT_SSL_PREFIX + sslProperty,
String.class);
if (property.isPresent()) {
return SCHEMA_HTTPS;
}
}

return SCHEMA_HTTP;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,16 @@

public class KubernetesWithHealthUsingManagementInterfaceTest {

private static final String NAME = "kubernetes-with-health-and-management";

@RegisterExtension
static final QuarkusProdModeTest config = new QuarkusProdModeTest()
.withApplicationRoot((jar) -> jar.addClasses(GreetingResource.class))
.setApplicationName("health")
.setApplicationName(NAME)
.setApplicationVersion("0.1-SNAPSHOT")
.setRun(true)
.setLogFileName("k8s.log")
.withConfigurationResource("kubernetes-with-health-and-management.properties")
.withConfigurationResource(NAME + ".properties")
.setForcedDependencies(List.of(
Dependency.of("io.quarkus", "quarkus-smallrye-health", Version.getVersion())));

Expand Down Expand Up @@ -64,7 +66,7 @@ public void assertGeneratedResources() throws IOException {
.deserializeAsList(kubernetesDir.resolve("kubernetes.yml"));
assertThat(kubernetesList.get(0)).isInstanceOfSatisfying(Deployment.class, d -> {
assertThat(d.getMetadata()).satisfies(m -> {
assertThat(m.getName()).isEqualTo("health");
assertThat(m.getName()).isEqualTo(NAME);
});

assertThat(d.getSpec()).satisfies(deploymentSpec -> {
Expand All @@ -76,14 +78,16 @@ public void assertGeneratedResources() throws IOException {
assertProbePath(p, "/q/health/ready");

assertNotNull(p.getHttpGet());
assertEquals(p.getHttpGet().getPort().getIntVal(), 9000);
assertEquals("HTTP", p.getHttpGet().getScheme());
assertEquals(9000, p.getHttpGet().getPort().getIntVal());
});
assertThat(container.getLivenessProbe()).isNotNull().satisfies(p -> {
assertThat(p.getInitialDelaySeconds()).isEqualTo(20);
assertProbePath(p, "/liveness");

assertNotNull(p.getHttpGet());
assertEquals(p.getHttpGet().getPort().getIntVal(), 9000);
assertEquals("HTTP", p.getHttpGet().getScheme());
assertEquals(9000, p.getHttpGet().getPort().getIntVal());
});
});
});
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package io.quarkus.it.kubernetes;

import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;

import java.io.IOException;
import java.nio.file.Path;
import java.util.List;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.Probe;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.quarkus.builder.Version;
import io.quarkus.maven.dependency.Dependency;
import io.quarkus.test.LogFile;
import io.quarkus.test.ProdBuildResults;
import io.quarkus.test.ProdModeTestResults;
import io.quarkus.test.QuarkusProdModeTest;

public class KubernetesWithHealthUsingSecuredManagementInterfaceAtRuntimeTest {

private static final String NAME = "kubernetes-with-health-and-secured-management-at-runtime";

@RegisterExtension
static final QuarkusProdModeTest config = new QuarkusProdModeTest()
.withApplicationRoot((jar) -> jar.addClasses(GreetingResource.class))
.setApplicationName(NAME)
.setApplicationVersion("0.1-SNAPSHOT")
.withConfigurationResource(NAME + ".properties")
.setForcedDependencies(List.of(
Dependency.of("io.quarkus", "quarkus-smallrye-health", Version.getVersion())));

@ProdBuildResults
private ProdModeTestResults prodModeTestResults;

@LogFile
private Path logfile;

@Test
public void assertGeneratedResources() throws IOException {
final Path kubernetesDir = prodModeTestResults.getBuildDir().resolve("kubernetes");
assertThat(kubernetesDir)
.isDirectoryContaining(p -> p.getFileName().endsWith("kubernetes.json"))
.isDirectoryContaining(p -> p.getFileName().endsWith("kubernetes.yml"));
List<HasMetadata> kubernetesList = DeserializationUtil
.deserializeAsList(kubernetesDir.resolve("kubernetes.yml"));
assertThat(kubernetesList.get(0)).isInstanceOfSatisfying(Deployment.class, d -> {
assertThat(d.getMetadata()).satisfies(m -> {
assertThat(m.getName()).isEqualTo(NAME);
});

assertThat(d.getSpec()).satisfies(deploymentSpec -> {
assertThat(deploymentSpec.getTemplate()).satisfies(t -> {
assertThat(t.getSpec()).satisfies(podSpec -> {
assertThat(podSpec.getContainers()).singleElement().satisfies(container -> {
assertThat(container.getReadinessProbe()).isNotNull().satisfies(p -> {
assertThat(p.getInitialDelaySeconds()).isEqualTo(5);
assertProbePath(p, "/q/health/ready");

assertNotNull(p.getHttpGet());
assertEquals("HTTPS", p.getHttpGet().getScheme());
assertEquals(9000, p.getHttpGet().getPort().getIntVal());
});
assertThat(container.getLivenessProbe()).isNotNull().satisfies(p -> {
assertThat(p.getInitialDelaySeconds()).isEqualTo(20);
assertProbePath(p, "/liveness");

assertNotNull(p.getHttpGet());
assertEquals("HTTPS", p.getHttpGet().getScheme());
assertEquals(9000, p.getHttpGet().getPort().getIntVal());
});
});
});
});
});
});
}

private void assertProbePath(Probe p, String expectedPath) {
assertThat(p.getHttpGet()).satisfies(h -> {
assertThat(h.getPath()).isEqualTo(expectedPath);
});
}
}
Loading

0 comments on commit 222933c

Please sign in to comment.