Merge branch 'master' into docs/update-lineage-feature-guide

.github/workflows/airflow-plugin.yml

@@ -51,6 +51,9 @@ jobs:
           - python-version: "3.10"
             extra_pip_requirements: 'apache-airflow==2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
+          - python-version: "3.10"
+            extra_pip_requirements: 'apache-airflow==2.9.0 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.10.txt'
+            extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
       - name: Set up JDK 17

.github/workflows/docker-unified.yml

@@ -6,6 +6,11 @@ on:
   pull_request:
     branches:
       - "**"
+    types:
+      - labeled
+      - opened
+      - synchronize
+      - reopened
   release:
     types: [published]
 
@@ -125,7 +130,8 @@ jobs:
         if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
         run: |
           python ./.github/scripts/check_python_package.py
-          ./gradlew :smoke-test:lint
+          ./gradlew :smoke-test:pythonLint
+          ./gradlew :smoke-test:cypressLint
 
   gms_build:
     name: Build and Push DataHub GMS Docker Image

README.md

@@ -28,7 +28,7 @@ HOSTED_DOCS_ONLY-->
 [![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub)
 [![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush)
 [![Docker Pulls](https://img.shields.io/docker/pulls/acryldata/datahub-gms.svg)](https://hub.docker.com/r/acryldata/datahub-gms)
-[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://datahubproject.io/slack?utm_source=docs&utm_medium=docs&utm_campaign=docs_page_link)
+[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md)
 [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr)
 [![License](https://img.shields.io/github/license/datahub-project/datahub)](https://github.com/datahub-project/datahub/blob/master/LICENSE)
@@ -106,7 +106,7 @@ We welcome contributions from the community. Please refer to our [Contributing G
 
 ## Community
 
-Join our [Slack workspace](https://datahubproject.io/slack?utm_source=docs&utm_medium=docs&utm_campaign=docs_page_link) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
+Join our [Slack workspace](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
 
 ## Adoption
 

build.gradle

@@ -54,7 +54,7 @@ buildscript {
   ext.hazelcastVersion = '5.3.6'
   ext.ebeanVersion = '12.16.1'
   ext.googleJavaFormatVersion = '1.18.1'
-  ext.openLineageVersion = '1.5.0'
+  ext.openLineageVersion = '1.13.1'
   ext.logbackClassicJava8 = '1.2.12'
 
   ext.docker_registry = 'acryldata'
@@ -168,7 +168,8 @@ project.ext.externalDependency = [
     'jettison': 'org.codehaus.jettison:jettison:1.5.4',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
     'jna': 'net.java.dev.jna:jna:5.12.1',
-    'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
+    'jsonPatch': 'jakarta.json:jakarta.json-api:2.1.3',
+    'jsonPathImpl': 'org.eclipse.parsson:parsson:1.1.6',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20231013',
@@ -271,7 +272,7 @@ project.ext.externalDependency = [
     'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0',
     'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2',
     'jakartaAnnotationApi': 'jakarta.annotation:jakarta.annotation-api:3.0.0',
-    'classGraph': 'io.github.classgraph:classgraph:4.8.168',
+    'classGraph': 'io.github.classgraph:classgraph:4.8.172',
 ]
 
 allprojects {

datahub-frontend/app/auth/AuthModule.java

@@ -62,6 +62,7 @@ public class AuthModule extends AbstractModule {
   private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider";
   private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval";
   private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries";
+  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize";
   private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings";
 
   private final com.typesafe.config.Config _configs;
@@ -201,11 +202,13 @@ protected ConfigurationProvider provideConfigurationProvider() {
   protected SystemEntityClient provideEntityClient(
       @Named("systemOperationContext") final OperationContext systemOperationContext,
       final ConfigurationProvider configurationProvider) {
+
     return new SystemRestliEntityClient(
         buildRestliClient(),
         new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
         _configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
-        configurationProvider.getCache().getClient().getEntityClient());
+        configurationProvider.getCache().getClient().getEntityClient(),
+        Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)));
   }
 
   @Provides

datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java

@@ -211,6 +211,8 @@ private Result handleOidcCallback(
                 "Failed to perform post authentication steps. Error message: %s", e.getMessage()));
       }
 
+      log.info("OIDC callback authentication successful for user: {}", userName);
+
       // Successfully logged in - Generate GMS login token
       final String accessToken = authClient.generateSessionTokenForUser(corpUserUrn.getId());
       return result

datahub-frontend/app/client/AuthServiceClient.java

@@ -75,7 +75,6 @@ public String generateSessionTokenForUser(@Nonnull final String userId) {
     CloseableHttpResponse response = null;
 
     try {
-
       final String protocol = this.metadataServiceUseSsl ? "https" : "http";
       final HttpPost request =
           new HttpPost(
@@ -86,6 +85,8 @@ public String generateSessionTokenForUser(@Nonnull final String userId) {
                   this.metadataServicePort,
                   GENERATE_SESSION_TOKEN_ENDPOINT));
 
+      log.info("Requesting session token for user: {}", userId);
+
       // Build JSON request to generate a token on behalf of a user.
       final ObjectMapper objectMapper = new ObjectMapper();
       final ObjectNode objectNode = objectMapper.createObjectNode();
@@ -100,7 +101,7 @@ public String generateSessionTokenForUser(@Nonnull final String userId) {
       response = httpClient.execute(request);
       final HttpEntity entity = response.getEntity();
       if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && entity != null) {
-        // Successfully generated a token for the User
+        log.info("Successfully received session token for user: {}", userId);
         final String jsonStr = EntityUtils.toString(entity);
         return getAccessTokenFromJson(jsonStr);
       } else {
@@ -110,6 +111,7 @@ public String generateSessionTokenForUser(@Nonnull final String userId) {
                 response.getStatusLine().toString(), response.getEntity().toString()));
       }
     } catch (Exception e) {
+      log.error("Failed to generate session token for user: {}", userId, e);
       throw new RuntimeException("Failed to generate session token for user", e);
     } finally {
       try {

datahub-frontend/app/controllers/AuthenticationController.java

@@ -42,7 +42,6 @@
 import play.mvc.Results;
 import security.AuthenticationManager;
 
-// TODO add logging.
 public class AuthenticationController extends Controller {
   public static final String AUTH_VERBOSE_LOGGING = "auth.verbose.logging";
   private static final String AUTH_REDIRECT_URI_PARAM = "redirect_uri";
@@ -183,10 +182,12 @@ public Result logIn(Http.Request request) {
     boolean loginSucceeded = tryLogin(username, password);
 
     if (!loginSucceeded) {
+      _logger.info("Login failed for user: {}", username);
       return Results.badRequest(invalidCredsJson);
     }
 
     final Urn actorUrn = new CorpuserUrn(username);
+    _logger.info("Login successful for user: {}, urn: {}", username, actorUrn);
     final String accessToken = _authClient.generateSessionTokenForUser(actorUrn.getId());
     return createSession(actorUrn.toString(), accessToken);
   }
@@ -250,6 +251,7 @@ public Result signUp(Http.Request request) {
     final Urn userUrn = new CorpuserUrn(email);
     final String userUrnString = userUrn.toString();
     _authClient.signUp(userUrnString, fullName, email, title, password, inviteToken);
+    _logger.info("Signed up user {} using invite tokens", userUrnString);
     final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId());
     return createSession(userUrnString, accessToken);
   }
@@ -351,15 +353,15 @@ private boolean tryLogin(String username, String password) {
     // First try jaas login, if enabled
     if (_jaasConfigs.isJAASEnabled()) {
       try {
-        _logger.debug("Attempting jaas authentication");
+        _logger.debug("Attempting JAAS authentication for user: {}", username);
         AuthenticationManager.authenticateJaasUser(username, password);
-        _logger.debug("Jaas authentication successful. Login succeeded");
+        _logger.debug("JAAS authentication successful. Login succeeded");
         loginSucceeded = true;
       } catch (Exception e) {
         if (_verbose) {
-          _logger.debug("Jaas authentication error. Login failed", e);
+          _logger.debug("JAAS authentication error. Login failed", e);
         } else {
-          _logger.debug("Jaas authentication error. Login failed");
+          _logger.debug("JAAS authentication error. Login failed");
         }
       }
     }

datahub-frontend/conf/application.conf

@@ -288,4 +288,6 @@ systemClientSecret=${?DATAHUB_SYSTEM_CLIENT_SECRET}
 entityClient.retryInterval = 2
 entityClient.retryInterval = ${?ENTITY_CLIENT_RETRY_INTERVAL}
 entityClient.numRetries = 3
-entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES}
+entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES}
+entityClient.restli.get.batchSize = 100
+entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -21,6 +21,7 @@ private Constants() {}
   public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql";
   public static final String FORMS_SCHEMA_FILE = "forms.graphql";
   public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql";
+  public static final String CONNECTIONS_SCHEMA_FILE = "connection.graphql";
   public static final String BROWSE_PATH_DELIMITER = "/";
   public static final String BROWSE_PATH_V2_DELIMITER = "␟";
   public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -48,6 +48,7 @@
 import com.linkedin.datahub.graphql.generated.DashboardStatsSummary;
 import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts;
 import com.linkedin.datahub.graphql.generated.DataFlow;
+import com.linkedin.datahub.graphql.generated.DataHubConnection;
 import com.linkedin.datahub.graphql.generated.DataHubView;
 import com.linkedin.datahub.graphql.generated.DataJob;
 import com.linkedin.datahub.graphql.generated.DataJobInputOutput;
@@ -129,6 +130,7 @@
 import com.linkedin.datahub.graphql.resolvers.chart.BrowseV2Resolver;
 import com.linkedin.datahub.graphql.resolvers.chart.ChartStatsSummaryResolver;
 import com.linkedin.datahub.graphql.resolvers.config.AppConfigResolver;
+import com.linkedin.datahub.graphql.resolvers.connection.UpsertConnectionResolver;
 import com.linkedin.datahub.graphql.resolvers.container.ContainerEntitiesResolver;
 import com.linkedin.datahub.graphql.resolvers.container.ParentContainersResolver;
 import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardStatsSummaryResolver;
@@ -306,6 +308,7 @@
 import com.linkedin.datahub.graphql.types.chart.ChartType;
 import com.linkedin.datahub.graphql.types.common.mappers.OperationMapper;
 import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
+import com.linkedin.datahub.graphql.types.connection.DataHubConnectionType;
 import com.linkedin.datahub.graphql.types.container.ContainerType;
 import com.linkedin.datahub.graphql.types.corpgroup.CorpGroupType;
 import com.linkedin.datahub.graphql.types.corpuser.CorpUserType;
@@ -355,6 +358,7 @@
 import com.linkedin.metadata.config.ViewsConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
 import com.linkedin.metadata.config.telemetry.TelemetryConfiguration;
+import com.linkedin.metadata.connection.ConnectionService;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.graph.GraphClient;
 import com.linkedin.metadata.graph.SiblingGraphService;
@@ -439,6 +443,7 @@ public class GmsGraphQLEngine {
   private final ERModelRelationshipService erModelRelationshipService;
   private final FormService formService;
   private final RestrictedService restrictedService;
+  private ConnectionService connectionService;
 
   private final BusinessAttributeService businessAttributeService;
   private final FeatureFlags featureFlags;
@@ -472,6 +477,7 @@ public class GmsGraphQLEngine {
   private final GlossaryTermType glossaryTermType;
   private final GlossaryNodeType glossaryNodeType;
   private final AspectType aspectType;
+  private final DataHubConnectionType connectionType;
   private final ContainerType containerType;
   private final DomainType domainType;
   private final NotebookType notebookType;
@@ -497,6 +503,7 @@ public class GmsGraphQLEngine {
 
   private final int graphQLQueryComplexityLimit;
   private final int graphQLQueryDepthLimit;
+  private final boolean graphQLQueryIntrospectionEnabled;
 
   private final BusinessAttributeType businessAttributeType;
 
@@ -557,6 +564,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
     this.dataProductService = args.dataProductService;
     this.formService = args.formService;
     this.restrictedService = args.restrictedService;
+    this.connectionService = args.connectionService;
 
     this.businessAttributeService = args.businessAttributeService;
     this.ingestionConfiguration = Objects.requireNonNull(args.ingestionConfiguration);
@@ -587,6 +595,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
     this.glossaryTermType = new GlossaryTermType(entityClient);
     this.glossaryNodeType = new GlossaryNodeType(entityClient);
     this.aspectType = new AspectType(entityClient);
+    this.connectionType = new DataHubConnectionType(entityClient, secretService);
     this.containerType = new ContainerType(entityClient);
     this.domainType = new DomainType(entityClient);
     this.notebookType = new NotebookType(entityClient);
@@ -612,6 +621,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
 
     this.graphQLQueryComplexityLimit = args.graphQLQueryComplexityLimit;
     this.graphQLQueryDepthLimit = args.graphQLQueryDepthLimit;
+    this.graphQLQueryIntrospectionEnabled = args.graphQLQueryIntrospectionEnabled;
 
     this.businessAttributeType = new BusinessAttributeType(entityClient);
     // Init Lists
@@ -634,6 +644,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
             dataJobType,
             glossaryTermType,
             glossaryNodeType,
+            connectionType,
             containerType,
             notebookType,
             domainType,
@@ -751,6 +762,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
     configureRoleResolvers(builder);
     configureBusinessAttributeResolver(builder);
     configureBusinessAttributeAssociationResolver(builder);
+    configureConnectionResolvers(builder);
   }
 
   private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) {
@@ -801,6 +813,7 @@ public GraphQLEngine.Builder builder() {
         .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE))
         .addSchema(fileBasedSchema(PROPERTIES_SCHEMA_FILE))
         .addSchema(fileBasedSchema(FORMS_SCHEMA_FILE))
+        .addSchema(fileBasedSchema(CONNECTIONS_SCHEMA_FILE))
         .addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE));
 
     for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
@@ -819,7 +832,8 @@ public GraphQLEngine.Builder builder() {
         .addDataLoader("Aspect", context -> createDataLoader(aspectType, context))
         .configureRuntimeWiring(this::configureRuntimeWiring)
         .setGraphQLQueryComplexityLimit(graphQLQueryComplexityLimit)
-        .setGraphQLQueryDepthLimit(graphQLQueryDepthLimit);
+        .setGraphQLQueryDepthLimit(graphQLQueryDepthLimit)
+        .setGraphQLQueryIntrospectionEnabled(graphQLQueryIntrospectionEnabled);
     return builder;
   }
 
@@ -3012,4 +3026,29 @@ private void configureBusinessAttributeAssociationResolver(final RuntimeWiring.B
                             .getBusinessAttribute()
                             .getUrn())));
   }
+
+  private void configureConnectionResolvers(final RuntimeWiring.Builder builder) {
+    builder.type(
+        "Mutation",
+        typeWiring ->
+            typeWiring.dataFetcher(
+                "upsertConnection",
+                new UpsertConnectionResolver(connectionService, secretService)));
+    builder.type(
+        "Query",
+        typeWiring -> typeWiring.dataFetcher("connection", getResolver(this.connectionType)));
+    builder.type(
+        "DataHubConnection",
+        typeWiring ->
+            typeWiring.dataFetcher(
+                "platform",
+                new LoadableTypeResolver<>(
+                    this.dataPlatformType,
+                    (env) -> {
+                      final DataHubConnection connection = env.getSource();
+                      return connection.getPlatform() != null
+                          ? connection.getPlatform().getUrn()
+                          : null;
+                    })));
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java

@@ -19,6 +19,7 @@
 import com.linkedin.metadata.config.ViewsConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
 import com.linkedin.metadata.config.telemetry.TelemetryConfiguration;
+import com.linkedin.metadata.connection.ConnectionService;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.graph.GraphClient;
 import com.linkedin.metadata.graph.SiblingGraphService;
@@ -82,7 +83,9 @@ public class GmsGraphQLEngineArgs {
   RestrictedService restrictedService;
   int graphQLQueryComplexityLimit;
   int graphQLQueryDepthLimit;
+  boolean graphQLQueryIntrospectionEnabled;
   BusinessAttributeService businessAttributeService;
+  ConnectionService connectionService;
 
   // any fork specific args should go below this line
 }