Skip to content

Commit

Permalink
allow using http2
Browse files Browse the repository at this point in the history
  • Loading branch information
annie-mac committed Nov 14, 2024
1 parent 6556422 commit 128a645
Show file tree
Hide file tree
Showing 8 changed files with 191 additions and 16 deletions.
8 changes: 8 additions & 0 deletions sdk/cosmos/azure-cosmos/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,14 @@ Licensed under the MIT License.
</rules>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
import com.azure.cosmos.implementation.circuitBreaker.PartitionLevelCircuitBreakerConfig;
import com.azure.cosmos.implementation.directconnectivity.Protocol;
import io.netty.handler.codec.http2.Http2SecurityUtil;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.ApplicationProtocolNames;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.SupportedCipherSuiteFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -32,7 +35,7 @@ public class Configs {
public static final String SPECULATION_THRESHOLD = "COSMOS_SPECULATION_THRESHOLD";
public static final String SPECULATION_THRESHOLD_STEP = "COSMOS_SPECULATION_THRESHOLD_STEP";
private final SslContext sslContext;

private final SslContext sslContextForHttp2;
// The names we use are consistent with the:
// * Azure environment variable naming conventions documented at https://azure.github.io/azure-sdk/java_implementation.html and
// * Java property naming conventions as illustrated by the name/value pairs returned by System.getProperties.
Expand Down Expand Up @@ -255,19 +258,41 @@ public class Configs {
private static final boolean DEFAULT_PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN = false;
private static final String PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN = "COSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN";

// Flag to indicate whether enabled http2 for gateway, Please do not use it, only for internal testing purpose
private static final boolean DEFAULT_USE_HTTP2 = false;
private static final String USE_HTTP2 = "COSMOS.USE_HTTP2";
private static final String USE_HTTP2_VARIABLE = "COSMOS_USE_HTTP2";

public Configs() {
this.sslContext = sslContextInit();
this.sslContext = sslContextInit(false);
this.sslContextForHttp2 = sslContextInit(true);
}

public static int getCPUCnt() {
return CPU_CNT;
}

private SslContext sslContextInit() {
private SslContext sslContextInit(boolean useHttp2) {
try {
SslProvider sslProvider = SslContext.defaultClientProvider();
return SslContextBuilder.forClient().sslProvider(sslProvider).build();
SslContextBuilder sslContextBuilder =
SslContextBuilder
.forClient()
.sslProvider(SslContext.defaultClientProvider());

if (useHttp2) {
sslContextBuilder
.ciphers(Http2SecurityUtil.CIPHERS, SupportedCipherSuiteFilter.INSTANCE)
.applicationProtocolConfig(
new ApplicationProtocolConfig(
ApplicationProtocolConfig.Protocol.ALPN,
ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,
ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,
ApplicationProtocolNames.HTTP_2
)
);
}

return sslContextBuilder.build();
} catch (SSLException sslException) {
logger.error("Fatal error cannot instantiate ssl context due to {}", sslException.getMessage(), sslException);
throw new IllegalStateException(sslException);
Expand All @@ -278,6 +303,10 @@ public SslContext getSslContext() {
return this.sslContext;
}

public SslContext getSslContextForHttp2() {
return this.sslContextForHttp2;
}

public Protocol getProtocol() {
String protocol = System.getProperty(PROTOCOL_PROPERTY, firstNonNull(
emptyToNull(System.getenv().get(PROTOCOL_ENVIRONMENT_VARIABLE)),
Expand Down Expand Up @@ -813,4 +842,14 @@ public static String getCharsetDecoderErrorActionOnUnmappedCharacter() {
emptyToNull(System.getenv().get(CHARSET_DECODER_ERROR_ACTION_ON_UNMAPPED_CHARACTER)),
DEFAULT_CHARSET_DECODER_ERROR_ACTION_ON_UNMAPPED_CHARACTER));
}

public static boolean shouldUseHttp2() {
String httpEnabledConfig = System.getProperty(
USE_HTTP2,
firstNonNull(
emptyToNull(System.getenv().get(USE_HTTP2_VARIABLE)),
String.valueOf(DEFAULT_USE_HTTP2)));

return Boolean.parseBoolean(httpEnabledConfig);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@
import reactor.core.publisher.SignalType;
import reactor.util.concurrent.Queues;
import reactor.util.function.Tuple2;
import reactor.util.retry.Retry;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
Expand Down Expand Up @@ -672,9 +671,6 @@ private void updateGatewayProxy() {

public void init(CosmosClientMetadataCachesSnapshot metadataCachesSnapshot, Function<HttpClient, HttpClient> httpClientInterceptor) {
try {
// TODO: add support for openAsync
// https://msdata.visualstudio.com/CosmosDB/_workitems/edit/332589

this.httpClientInterceptor = httpClientInterceptor;
if (httpClientInterceptor != null) {
this.reactorHttpClient = httpClientInterceptor.apply(httpClient());
Expand Down Expand Up @@ -835,7 +831,8 @@ private HttpClient httpClient() {
.withMaxIdleConnectionTimeout(this.connectionPolicy.getIdleHttpConnectionTimeout())
.withPoolSize(this.connectionPolicy.getMaxConnectionPoolSize())
.withProxy(this.connectionPolicy.getProxy())
.withNetworkRequestTimeout(this.connectionPolicy.getHttpNetworkRequestTimeout());
.withNetworkRequestTimeout(this.connectionPolicy.getHttpNetworkRequestTimeout())
.withHttp2(Configs.shouldUseHttp2());

if (connectionSharingAcrossClientsEnabled) {
return SharedGatewayHttpClient.getOrCreateInstance(httpClientConfig, diagnosticsClientConfig);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.azure.cosmos.implementation.http;

import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.codec.http2.Http2Headers;
import io.netty.handler.codec.http2.Http2HeadersFrame;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class Http2ResponseHeaderCleanerHandler extends ChannelInboundHandlerAdapter {

private static final Logger logger = LoggerFactory.getLogger(Http2ResponseHeaderCleanerHandler.class);
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
logger.info("In Http2HeaderCleanerHandler ");
if (msg instanceof Http2HeadersFrame) {
Http2HeadersFrame headersFrame = (Http2HeadersFrame) msg;
Http2Headers headers = headersFrame.headers();

headers.forEach(entry -> {
CharSequence key = entry.getKey();
CharSequence value = entry.getValue();

// Check for leading whitespace or other prohibited characters
if (StringUtils.isNotEmpty(value) && (value.charAt(0) == ' ' || value.charAt(value.length() - 1) == ' ')) {
// Clean up the header value by trimming or handling as needed
logger.warn("There are extra white space for key {} with value {}", key, value);

// TODO[Http2]: for now just trim the spaces, explore other options for example escape the whitespace
headers.set(key, value.toString().trim());
}
});
}

// Pass the message to the next handler in the pipeline
super.channelRead(ctx, msg);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class HttpClientConfig {
private Duration networkRequestTimeout;
private ProxyOptions proxy;
private boolean connectionKeepAlive = true;
private boolean useHttp2;

public HttpClientConfig(Configs configs) {
this.configs = configs;
Expand Down Expand Up @@ -51,6 +52,11 @@ public HttpClientConfig withConnectionKeepAlive(boolean connectionKeepAlive) {
return this;
}

public HttpClientConfig withHttp2(boolean useHttp2) {
this.useHttp2 = useHttp2;
return this;
}

public Configs getConfigs() {
return configs;
}
Expand All @@ -75,13 +81,18 @@ public boolean isConnectionKeepAlive() {
return connectionKeepAlive;
}

public boolean shouldUseHttp2() {
return useHttp2;
}

// TODO(kuthapar): Do we really need to use Strings.lenientFormat() here?
// Even the documentation of this API suggests to use String.format or just string appends if possible.
public String toDiagnosticsString() {
return Strings.lenientFormat("(cps:%s, nrto:%s, icto:%s, p:%s)",
return Strings.lenientFormat("(cps:%s, nrto:%s, icto:%s, p:%s, http2:%s)",
maxPoolSize,
networkRequestTimeout,
maxIdleConnectionTimeout,
proxy != null);
proxy != null,
useHttp2);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.azure.cosmos.implementation.Configs;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.logging.LogLevel;
import io.netty.resolver.DefaultAddressResolverGroup;
Expand All @@ -17,6 +18,7 @@
import reactor.netty.Connection;
import reactor.netty.ConnectionObserver;
import reactor.netty.NettyOutbound;
import reactor.netty.http.HttpProtocol;
import reactor.netty.http.client.HttpClientRequest;
import reactor.netty.http.client.HttpClientResponse;
import reactor.netty.http.client.HttpClientState;
Expand Down Expand Up @@ -76,7 +78,7 @@ public static ReactorNettyClient create(HttpClientConfig httpClientConfig) {
.newConnection()
.observe(getConnectionObserver())
.resolver(DefaultAddressResolverGroup.INSTANCE);
reactorNettyClient.configureChannelPipelineHandlers();
reactorNettyClient.configureChannelPipelineHandlers(httpClientConfig.shouldUseHttp2());
attemptToWarmupHttpClient(reactorNettyClient);
return reactorNettyClient;
}
Expand All @@ -92,7 +94,7 @@ public static ReactorNettyClient createWithConnectionProvider(ConnectionProvider
.create(connectionProvider)
.observe(getConnectionObserver())
.resolver(DefaultAddressResolverGroup.INSTANCE);
reactorNettyClient.configureChannelPipelineHandlers();
reactorNettyClient.configureChannelPipelineHandlers(httpClientConfig.shouldUseHttp2());
attemptToWarmupHttpClient(reactorNettyClient);
return reactorNettyClient;
}
Expand All @@ -117,7 +119,7 @@ private static void attemptToWarmupHttpClient(ReactorNettyClient reactorNettyCli
}
}

private void configureChannelPipelineHandlers() {
private void configureChannelPipelineHandlers(boolean useHttp2) {
Configs configs = this.httpClientConfig.getConfigs();

if (this.httpClientConfig.getProxy() != null) {
Expand All @@ -139,6 +141,22 @@ private void configureChannelPipelineHandlers() {
.maxHeaderSize(configs.getMaxHttpHeaderSize())
.maxChunkSize(configs.getMaxHttpChunkSize())
.validateHeaders(true));

if (useHttp2) {
this.httpClient = this.httpClient
.secure(sslContextSpec -> sslContextSpec.sslContext(configs.getSslContextForHttp2()))
.protocol(HttpProtocol.H2)
.doOnChannelInit((connectionObserver, channel, remoteAddress) -> {
// The response header clean up pipeline is being added due to an error getting when calling gateway:
// java.lang.IllegalArgumentException: a header value contains prohibited character 0x20 at index 0 for 'x-ms-serviceversion', there is whitespace in the front of the value.
// validateHeaders(false) does not work for http2
ChannelPipeline channelPipeline = channel.pipeline();
channelPipeline.addAfter(
"reactor.left.httpCodec",
"customHeaderCleaner",
new Http2ResponseHeaderCleanerHandler());
});
}
}

@Override
Expand Down
27 changes: 27 additions & 0 deletions sdk/cosmos/live-http2-platform-matrix.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
"displayNames": {
"-Pfast": "Fast",
"-Pquery": "Query",
"-Pcircuit-breaker-misc-gateway": "CircuitBreakerMiscGateway",
"Session": "",
"ubuntu": "",
"@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }": ""
},
"include": [
{
"DESIRED_CONSISTENCIES": "[\"Session\"]",
"ACCOUNT_CONSISTENCY": "Session",
"ArmConfig": {
"MultiMaster_MultiRegion": {
"ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }",
"PREFERRED_LOCATIONS": "[\"East US 2\"]"
}
},
"PROTOCOLS": "[\"Tcp\"]",
"ProfileFlag": [ "-Pquery", "-Pcircuit-breaker-misc-gateway", "-Pfast" ],
"Agent": {
"ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }
}
}
]
}
33 changes: 33 additions & 0 deletions sdk/cosmos/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,39 @@ extends:
- name: AdditionalArgs
value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)'

- template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml
parameters:
CloudConfig:
Public:
ServiceConnection: azure-sdk-tests-cosmos
MatrixConfigs:
- Name: Cosmos_live_test_http2
Path: sdk/cosmos/live-http2-platform-matrix.json
Selection: all
GenerateVMJobs: true
MatrixReplace:
- .*Version=1.21/1.17
ServiceDirectory: cosmos
Artifacts:
- name: azure-cosmos
groupId: com.azure
safeName: azurecosmos
AdditionalModules:
- name: azure-cosmos-tests
groupId: com.azure
- name: azure-cosmos-benchmark
groupId: com.azure
TimeoutInMinutes: 210
MaxParallel: 20
PreSteps:
- template: /eng/pipelines/templates/steps/install-reporting-tools.yml
TestGoals: 'verify'
TestOptions: '$(ProfileFlag) $(AdditionalArgs) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false'
TestResultsFiles: '**/junitreports/TEST-*.xml'
AdditionalVariables:
- name: AdditionalArgs
value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DCOSMOS.USE_HTTP2=true'

- template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml
parameters:
TestName: 'Spring_Data_Cosmos_Integration'
Expand Down

0 comments on commit 128a645

Please sign in to comment.