Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for Firestore database id configuration #2164

Merged
merged 11 commits into from
Sep 19, 2023
2 changes: 1 addition & 1 deletion docs/src/main/asciidoc/datastore.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ The following configuration options are available:
| Name | Description | Required | Default value
| `spring.cloud.gcp.datastore.enabled` | Enables the Cloud Datastore client | No | `true`
| `spring.cloud.gcp.datastore.project-id` | Google Cloud project ID where the Google Cloud Datastore API is hosted, if different from the one in the <<spring-cloud-gcp-core,Spring Framework on Google Cloud Core Module>> | No |
| `spring.cloud.gcp.datastore.database-id` | Google Cloud project can host multiple databases. You can specify which database will be used. | No |
| `spring.cloud.gcp.datastore.database-id` | Google Cloud project can host multiple databases. You can specify which database will be used. If not specified, the database id will be "(default)". | No |
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"Default value" column is missing.

| `spring.cloud.gcp.datastore.credentials.location` | OAuth2 credentials for authenticating with the Google Cloud Datastore API, if different from the ones in the <<spring-cloud-gcp-core,Spring Framework on Google Cloud Core Module>> | No |
| `spring.cloud.gcp.datastore.credentials.encoded-key` | Base64-encoded OAuth2 credentials for authenticating with the Google Cloud Datastore API, if different from the ones in the <<spring-cloud-gcp-core,Spring Framework on Google Cloud Core Module>> | No |
| `spring.cloud.gcp.datastore.credentials.scopes` | https://developers.google.com/identity/protocols/googlescopes[OAuth2 scope] for Spring Framework on Google CloudDatastore credentials | No | https://www.googleapis.com/auth/datastore
Expand Down
1 change: 1 addition & 0 deletions docs/src/main/asciidoc/firestore.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ The Spring Boot starter for Google Cloud Firestore provides the following config
| Name | Description | Required | Default value
| `spring.cloud.gcp.firestore.enabled` | Enables or disables Firestore auto-configuration | No | `true`
| `spring.cloud.gcp.firestore.project-id` | Google Cloud project ID where the Google Cloud Firestore API is hosted, if different from the one in the <<spring-cloud-gcp-core,Spring Framework on Google Cloud Core Module>> | No |
| `spring.cloud.gcp.firestore.database-id` | Google Cloud project can host multiple databases. You can specify which database will be used. If not specified, the database id will be "(default)". | No |
| `spring.cloud.gcp.firestore.emulator.enabled` | Enables the usage of an emulator. If this is set to true, then you should set the `spring.cloud.gcp.firestore.host-port` to the host:port of your locally running emulator instance | No | `false`
| `spring.cloud.gcp.firestore.host-port` | The host and port of the Firestore service; can be overridden to specify connecting to an already-running https://firebase.google.com/docs/emulator-suite/install_and_configure[Firestore emulator] instance. | No | `firestore.googleapis.com:443` (the host/port of official Firestore service)
| `spring.cloud.gcp.firestore.credentials.location` | OAuth2 credentials for authenticating with the Google Cloud Firestore API, if different from the ones in the <<spring-cloud-gcp-core,Spring Framework on Google Cloud Core Module>> | No |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.google.api.gax.core.CredentialsProvider;
import com.google.api.gax.core.NoCredentialsProvider;
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
import com.google.api.gax.rpc.internal.Headers;
import com.google.cloud.firestore.Firestore;
import com.google.cloud.firestore.FirestoreOptions;
import com.google.cloud.spring.autoconfigure.core.GcpContextAutoConfiguration;
Expand All @@ -30,10 +31,15 @@
import com.google.cloud.spring.data.firestore.mapping.FirestoreDefaultClassMapper;
import com.google.cloud.spring.data.firestore.mapping.FirestoreMappingContext;
import com.google.firestore.v1.FirestoreGrpc;
import io.grpc.ClientInterceptor;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import io.grpc.Metadata;
import io.grpc.auth.MoreCallCredentials;
import io.grpc.stub.MetadataUtils;
import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
Expand Down Expand Up @@ -61,6 +67,8 @@ public class GcpFirestoreAutoConfiguration {

private final String projectId;

private final String databaseId;

private final CredentialsProvider credentialsProvider;

private final String hostPort;
Expand All @@ -74,6 +82,7 @@ public class GcpFirestoreAutoConfiguration {
throws IOException {

this.projectId = gcpFirestoreProperties.getResolvedProjectId(projectIdProvider);
this.databaseId = gcpFirestoreProperties.getResolvedDatabaseId();

if (gcpFirestoreProperties.getEmulator().isEnabled()) {
// if the emulator is enabled, create CredentialsProvider for this particular case.
Expand All @@ -95,6 +104,7 @@ public FirestoreOptions firestoreOptions() {
return FirestoreOptions.getDefaultInstance().toBuilder()
.setCredentialsProvider(this.credentialsProvider)
.setProjectId(this.projectId)
.setDatabaseId(databaseId)
.setHeaderProvider(USER_AGENT_HEADER_PROVIDER)
.setChannelProvider(
InstantiatingGrpcChannelProvider.newBuilder().setEndpoint(this.hostPort).build())
Expand Down Expand Up @@ -150,12 +160,27 @@ public FirestoreTemplate firestoreTemplate(
firestoreMappingContext);
}

@Bean
@ConditionalOnMissingBean(name = "firestoreRoutingHeadersInterceptor")
public ClientInterceptor firestoreRoutingHeadersInterceptor() {
// add routing header for custom database id
Metadata routingHeader = new Metadata();
Metadata.Key<String> key =
Metadata.Key.of(Headers.DYNAMIC_ROUTING_HEADER_KEY, Metadata.ASCII_STRING_MARSHALLER);
routingHeader.put(key,
Copy link
Contributor

@blakeli0 blakeli0 Sep 19, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This may work for this specific case, but routing header config could change in the future. I'm not sure it works for all the new routing header configs either, but maybe those are not required.
In long term, switching to GAPIC stubs should solve this issue, but I'm not sure it is compatible with the Reactive paradigm. Why did we choose to use the gRPC generated stubs instead of GAPIC ones? Only because of Reactive?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, this integration was specifically made to support reactive, and I believe at the time GAPIC was not compatible.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. My concern is that every time the backend makes the routing header required for a new RPC, or change how the routing header is constructed, we would have to come back and manually change it in spring-cloud-gcp. It may not in scope of this PR, can we investigate the possibility of switching to the GAPIC stubs?

"project_id=" + URLEncoder.encode(projectId, StandardCharsets.US_ASCII)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use PercentEscaper in gax, which I think has slight advantage over the Java default one based on @lqiu96's investigation.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Switch to `PercentEscaper1.

+ "&database_id=" + URLEncoder.encode(databaseId, StandardCharsets.US_ASCII));
return MetadataUtils.newAttachHeadersInterceptor(routingHeader);
}

@Bean
@ConditionalOnMissingBean(name = "firestoreManagedChannel")
public ManagedChannel firestoreManagedChannel() {
public ManagedChannel firestoreManagedChannel(
ClientInterceptor firestoreRoutingHeadersInterceptor) {
return ManagedChannelBuilder.forTarget(
"dns:///" + GcpFirestoreAutoConfiguration.this.hostPort)
.userAgent(USER_AGENT_HEADER_PROVIDER.getUserAgent())
.intercept(firestoreRoutingHeadersInterceptor)
.build();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public class GcpFirestoreEmulatorAutoConfiguration {
GcpFirestoreEmulatorAutoConfiguration(GcpFirestoreProperties properties) {
this.hostPort = properties.getHostPort();
this.projectId = StringUtils.defaultIfEmpty(properties.getProjectId(), "unused");
this.rootPath = String.format("projects/%s/databases/(default)", this.projectId);
this.rootPath = properties.getFirestoreRootPath(() -> projectId);
}

@Bean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
*/
@ConfigurationProperties("spring.cloud.gcp.firestore")
public class GcpFirestoreProperties implements CredentialsSupplier {
private static final String ROOT_PATH_FORMAT = "projects/%s/databases/(default)/documents";
private static final String ROOT_PATH_FORMAT = "projects/%s/databases/%s/documents";

/**
* Overrides the GCP OAuth2 credentials specified in the Core module. Uses same URL as Datastore
Expand All @@ -40,6 +40,8 @@ public class GcpFirestoreProperties implements CredentialsSupplier {

private String projectId;

private String databaseId;

/**
* The host and port of the Firestore emulator service; can be overridden to specify an emulator.
*/
Expand All @@ -65,6 +67,18 @@ public void setProjectId(String projectId) {
this.projectId = projectId;
}

public String getDatabaseId() {
return databaseId;
}

public String getResolvedDatabaseId() {
return this.getDatabaseId() == null ? "(default)" : this.getDatabaseId();
}

public void setDatabaseId(String databaseId) {
this.databaseId = databaseId;
}

public String getHostPort() {
return hostPort;
}
Expand All @@ -82,7 +96,8 @@ public void setEmulator(FirestoreEmulatorProperties emulator) {
}

public String getFirestoreRootPath(GcpProjectIdProvider projectIdProvider) {
return String.format(ROOT_PATH_FORMAT, this.getResolvedProjectId(projectIdProvider));
return String.format(ROOT_PATH_FORMAT, this.getResolvedProjectId(projectIdProvider),
this.getResolvedDatabaseId());
}

public static class FirestoreEmulatorProperties {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,19 @@
import com.google.cloud.firestore.FirestoreOptions;
import com.google.cloud.spring.autoconfigure.core.GcpContextAutoConfiguration;
import com.google.firestore.v1.FirestoreGrpc;
import io.grpc.ClientInterceptor;
import io.grpc.ManagedChannel;
import io.grpc.Metadata;
import io.grpc.Metadata.Key;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.NoSuchBeanDefinitionException;
import org.springframework.boot.autoconfigure.AutoConfigurationPackage;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.transaction.TransactionManager;

/**
Expand All @@ -55,11 +60,14 @@ class GcpFirestoreAutoConfigurationTests {
.withPropertyValues("spring.cloud.gcp.firestore.project-id=test-project");

@Test
void testDatastoreOptionsCorrectlySet() {
void testFirestoreOptionsCorrectlySet() {
this.contextRunner.run(
context -> {
FirestoreOptions datastoreOptions = context.getBean(Firestore.class).getOptions();
assertThat(datastoreOptions.getProjectId()).isEqualTo("test-project");
assertThat(datastoreOptions.getDatabaseId()).isEqualTo("(default)");
assertThat(getRoutingHeader(context))
.isEqualTo("project_id=test-project&database_id=%28default%29");
});
}

Expand All @@ -78,12 +86,21 @@ void testCorrectManagedChannel() {
.run(
ctx -> {
FirestoreGrpc.FirestoreStub stub =
(FirestoreGrpc.FirestoreStub) ctx.getBean("firestoreGrpcStub");
ctx.getBean("firestoreGrpcStub", FirestoreGrpc.FirestoreStub.class);
ManagedChannel channel = (ManagedChannel) stub.getChannel();
assertThat(channel.authority()).isEqualTo("firestore.googleapis.com:443");
assertThat(getRoutingHeader(ctx)).isEqualTo("project_id=test-project&database_id=%28default%29");
});
}

private static String getRoutingHeader(ApplicationContext ctx) {
ClientInterceptor clientInterceptor =
ctx.getBean("firestoreRoutingHeadersInterceptor", ClientInterceptor.class);
Metadata extraHeaders = (Metadata) ReflectionTestUtils.getField(clientInterceptor,
"extraHeaders");
return extraHeaders.get(Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER));
}

@Test
void testTransactionManagerExcludedWithoutAutoConfiguration() {
contextRunner
Expand All @@ -95,6 +112,24 @@ void testTransactionManagerExcludedWithoutAutoConfiguration() {
});
}

@Test
void testDatabaseIdOverride() {
contextRunner
.withPropertyValues("spring.cloud.gcp.firestore.database-id=mydb^2",
"spring.cloud.gcp.firestore.project-id=test")
.run(
context -> {
FirestoreOptions datastoreOptions = context.getBean(Firestore.class).getOptions();
assertThat(datastoreOptions.getDatabaseId()).isEqualTo("mydb^2");
String rootPath = context.getBean(GcpFirestoreProperties.class)
.getFirestoreRootPath(() -> "xyz-ignored");
assertThat(rootPath)
.isEqualTo("projects/test/databases/mydb^2/documents");
assertThat(getRoutingHeader(context)).isEqualTo(
"project_id=test&database_id=mydb%5E2");
});
}

/** Spring Boot config for tests. */
@AutoConfigurationPackage
static class TestConfiguration {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,6 @@ void testSortQuery_methodName_sortByDocumentId() {
@Configuration
@EnableReactiveFirestoreRepositories(basePackageClasses = UserRepository.class)
static class FirestoreRepositoryTestsConfiguration {
private static final String DEFAULT_PARENT =
"projects/my-project/databases/(default)/documents";

@Bean
public FirestoreMappingContext firestoreMappingContext() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
spring.cloud.gcp.firestore.project-id=spring-cloud-gcp-ci-firestore
spring.cloud.gcp.firestore.project-id=spring-cloud-gcp-ci
spring.cloud.gcp.firestore.database-id=firestoredb