Merge branch 'master' into container-platform-instance

datahub-project · Oct 13, 2022 · 68f8a0e · 68f8a0e
2 parents f178165 + 6e34cd6
commit 68f8a0e
Show file tree

Hide file tree

Showing 416 changed files with 25,473 additions and 23,023 deletions.
diff --git a/.github/pr-labeler-config.yml b/.github/pr-labeler-config.yml
@@ -9,6 +9,7 @@ product:
 - 'datahub-web-react/**/*'
 - 'datahub-frontend/**/*'
 - 'datahub-graphql-core/**/*'
+- 'metadata-io/**/*'
 
 docs:
 - 'docs/**/*'
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -25,6 +25,8 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:

diff --git a/build.gradle b/build.gradle
@@ -4,6 +4,10 @@ buildscript {
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.20'
   ext.springBootVersion = '2.5.12'
+  ext.openTelemetryVersion = '1.18.0'
+  ext.neo4jVersion = '4.4.9'
+  ext.graphQLJavaVersion = '19.0'
+  ext.testContainersVersion = '1.17.4'
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
   dependencies {
@@ -66,7 +70,8 @@ project.ext.externalDependency = [
     'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.3',
     'elasticSearchTransport': 'org.elasticsearch.client:transport:7.9.3',
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
-    'graphqlJava': 'com.graphql-java:graphql-java:16.1',
+    'graphqlJava': 'com.graphql-java:graphql-java:' + graphQLJavaVersion,
+    'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:' + graphQLJavaVersion,
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:4.2.2',
     'guava': 'com.google.guava:guava:27.0.1-jre',
@@ -88,6 +93,7 @@ project.ext.externalDependency = [
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': 'org.eclipse.jetty:jetty-jaas:9.4.46.v20220331',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
+    'jna': 'net.java.dev.jna:jna:5.12.1',
     'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
     'jsonSchemaAvro': 'com.github.fge:json-schema-avro:0.1.4',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
@@ -109,10 +115,10 @@ project.ext.externalDependency = [
     'mockServer': 'org.mock-server:mockserver-netty:5.11.2',
     'mockServerClient': 'org.mock-server:mockserver-client-java:5.11.2',
     'mysqlConnector': 'mysql:mysql-connector-java:8.0.20',
-    'neo4jHarness': 'org.neo4j.test:neo4j-harness:3.4.11',
-    'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:4.0.1',
-    'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:1.0.0',
-    'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:1.0.0',
+    'neo4jHarness': 'org.neo4j.test:neo4j-harness:' + neo4jVersion,
+    'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:' + neo4jVersion,
+    'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:' + openTelemetryVersion,
+    'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:' + openTelemetryVersion,
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
     'parquet': 'org.apache.parquet:parquet-avro:1.12.2',
     'picocli': 'info.picocli:picocli:4.5.0',
@@ -152,11 +158,11 @@ project.ext.externalDependency = [
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12',
     'testng': 'org.testng:testng:7.3.0',
-    'testContainers': 'org.testcontainers:testcontainers:1.15.3',
-    'testContainersJunit': 'org.testcontainers:junit-jupiter:1.15.3',
-    'testContainersPostgresql':'org.testcontainers:postgresql:1.15.3',
-    'testContainersElasticsearch': 'org.testcontainers:elasticsearch:1.15.3',
-    'testContainersCassandra': 'org.testcontainers:cassandra:1.15.3',
+    'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
+    'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion,
+    'testContainersPostgresql':'org.testcontainers:postgresql:' + testContainersVersion,
+    'testContainersElasticsearch': 'org.testcontainers:elasticsearch:' + testContainersVersion,
+    'testContainersCassandra': 'org.testcontainers:cassandra:' + testContainersVersion,
     'typesafeConfig':'com.typesafe:config:1.4.1',
     'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
     'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14'

diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java
@@ -1,19 +1,28 @@
 package auth;
 
+import auth.sso.SsoConfigs;
+import auth.sso.SsoManager;
+import auth.sso.oidc.OidcConfigs;
+import auth.sso.oidc.OidcProvider;
 import client.AuthServiceClient;
 import com.datahub.authentication.Actor;
 import com.datahub.authentication.ActorType;
+import com.datahub.authentication.Authentication;
 import com.google.inject.AbstractModule;
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.entity.client.RestliEntityClient;
 import com.linkedin.metadata.restli.DefaultRestliClientFactory;
 import com.linkedin.util.Configuration;
-import com.datahub.authentication.Authentication;
+import controllers.SsoCallbackController;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
 import org.pac4j.core.client.Client;
 import org.pac4j.core.client.Clients;
 import org.pac4j.core.config.Config;
@@ -25,14 +34,6 @@
 import org.pac4j.play.store.PlaySessionStore;
 import org.pac4j.play.store.ShiroAesDataEncrypter;
 import play.Environment;
-
-import java.util.ArrayList;
-import java.util.List;
-import auth.sso.oidc.OidcProvider;
-import auth.sso.oidc.OidcConfigs;
-import auth.sso.SsoConfigs;
-import auth.sso.SsoManager;
-import controllers.SsoCallbackController;
 import play.cache.SyncCacheApi;
 import utils.ConfigUtil;
 
@@ -142,44 +143,49 @@ protected SsoManager provideSsoManager() {
         return manager;
     }
 
-    @Provides @Singleton
+    @Provides
+    @Singleton
     protected Authentication provideSystemAuthentication() {
         // Returns an instance of Authentication used to authenticate system initiated calls to Metadata Service.
         String systemClientId = _configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH);
         String systemSecret = _configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH);
-        final Actor systemActor = new Actor(ActorType.USER, systemClientId); // TODO: Change to service actor once supported.
-        return new Authentication(
-            systemActor,
-            String.format("Basic %s:%s", systemClientId, systemSecret),
-            Collections.emptyMap()
-        );
+        final Actor systemActor =
+            new Actor(ActorType.USER, systemClientId); // TODO: Change to service actor once supported.
+        return new Authentication(systemActor, String.format("Basic %s:%s", systemClientId, systemSecret),
+            Collections.emptyMap());
     }
 
-    @Provides @Singleton
+    @Provides
+    @Singleton
     protected EntityClient provideEntityClient() {
         return new RestliEntityClient(buildRestliClient());
     }
 
-    @Provides @Singleton
-    protected AuthServiceClient provideAuthClient(Authentication systemAuthentication) {
+    @Provides
+    @Singleton
+    protected CloseableHttpClient provideHttpClient() {
+        return HttpClients.createDefault();
+    }
+
+    @Provides
+    @Singleton
+    protected AuthServiceClient provideAuthClient(Authentication systemAuthentication, CloseableHttpClient httpClient) {
         // Init a GMS auth client
-        final String metadataServiceHost = _configs.hasPath(METADATA_SERVICE_HOST_CONFIG_PATH)
-            ? _configs.getString(METADATA_SERVICE_HOST_CONFIG_PATH)
-            : Configuration.getEnvironmentVariable(GMS_HOST_ENV_VAR, DEFAULT_GMS_HOST);
-
-        final int metadataServicePort = _configs.hasPath(METADATA_SERVICE_PORT_CONFIG_PATH)
-            ? _configs.getInt(METADATA_SERVICE_PORT_CONFIG_PATH)
-            : Integer.parseInt(Configuration.getEnvironmentVariable(GMS_PORT_ENV_VAR, DEFAULT_GMS_PORT));
-
-        final Boolean metadataServiceUseSsl = _configs.hasPath(METADATA_SERVICE_USE_SSL_CONFIG_PATH)
-            ? _configs.getBoolean(METADATA_SERVICE_USE_SSL_CONFIG_PATH)
-            : Boolean.parseBoolean(Configuration.getEnvironmentVariable(GMS_USE_SSL_ENV_VAR, DEFAULT_GMS_USE_SSL));
-
-        return new AuthServiceClient(
-            metadataServiceHost,
-            metadataServicePort,
-            metadataServiceUseSsl,
-            systemAuthentication);
+        final String metadataServiceHost =
+            _configs.hasPath(METADATA_SERVICE_HOST_CONFIG_PATH) ? _configs.getString(METADATA_SERVICE_HOST_CONFIG_PATH)
+                : Configuration.getEnvironmentVariable(GMS_HOST_ENV_VAR, DEFAULT_GMS_HOST);
+
+        final int metadataServicePort =
+            _configs.hasPath(METADATA_SERVICE_PORT_CONFIG_PATH) ? _configs.getInt(METADATA_SERVICE_PORT_CONFIG_PATH)
+                : Integer.parseInt(Configuration.getEnvironmentVariable(GMS_PORT_ENV_VAR, DEFAULT_GMS_PORT));
+
+        final Boolean metadataServiceUseSsl =
+            _configs.hasPath(METADATA_SERVICE_USE_SSL_CONFIG_PATH) ? _configs.getBoolean(
+                METADATA_SERVICE_USE_SSL_CONFIG_PATH)
+                : Boolean.parseBoolean(Configuration.getEnvironmentVariable(GMS_USE_SSL_ENV_VAR, DEFAULT_GMS_USE_SSL));
+
+        return new AuthServiceClient(metadataServiceHost, metadataServicePort, metadataServiceUseSsl,
+            systemAuthentication, httpClient);
     }
 
     private com.linkedin.restli.client.Client buildRestliClient() {