Merge branch 'datahub-project:master' into master

.github/scripts/docker_helpers.sh

@@ -16,11 +16,11 @@ function get_tag {
 }
 
 function get_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
 }
 
 function get_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
 }
 
 function get_python_docker_release_v {
@@ -32,9 +32,9 @@ function get_unique_tag {
 }
 
 function get_unique_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
 }
 
 function get_unique_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
 }

.github/workflows/pr-labeler.yml

@@ -30,12 +30,10 @@ jobs:
                 "treff7es",
                 "yoonhyejin",
                 "eboneil",
-                "ethan-cartwright",
                 "gabe-lyons",
                 "hsheth2",
                 "jjoyce0510",
                 "maggiehays",
-                "mrjefflewis",
                 "pedro93",
                 "RyanHolstien",
                 "Kunal-kankriya",
@@ -45,7 +43,8 @@ jobs:
                 "kushagra-apptware",
                 "Salman-Apptware",
                 "mayurinehate",
-                "noggi"
+                "noggi",
+                "skrydal"
               ]'), 
               github.actor
             ) 
@@ -60,7 +59,6 @@ jobs:
           ${{ 
             contains(
               fromJson('[
-                "skrydal",
                 "siladitya2",
                 "sgomezvillamor",
                 "ngamanda",

datahub-frontend/app/controllers/SsoCallbackController.java

@@ -66,7 +66,8 @@ public SsoCallbackController(
 
   public CompletionStage<Result> handleCallback(String protocol, Http.Request request) {
     if (shouldHandleCallback(protocol)) {
-      log.debug(String.format("Handling SSO callback. Protocol: %s", protocol));
+      log.debug("Handling SSO callback. Protocol: {}",
+        _ssoManager.getSsoProvider().protocol().getCommonName());
       return callback(request)
           .handle(
               (res, e) -> {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -1032,6 +1032,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
                 .dataFetcher("mlModel", getResolver(mlModelType))
                 .dataFetcher("mlModelGroup", getResolver(mlModelGroupType))
                 .dataFetcher("assertion", getResolver(assertionType))
+                .dataFetcher("form", getResolver(formType))
+                .dataFetcher("view", getResolver(dataHubViewType))
                 .dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient))
                 .dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver())
                 .dataFetcher("listUsers", new ListUsersResolver(this.entityClient))

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java

@@ -41,7 +41,7 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
           final QueryContext context = environment.getContext();
           final String tokenId = bindArgument(environment.getArgument("tokenId"), String.class);
 
-          log.info("User {} revoking access token {}", context.getActorUrn(), tokenId);
+          log.info("User {} revoking access token", context.getActorUrn());
 
           if (isAuthorizedToRevokeToken(context, tokenId)) {
             try {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java

@@ -23,16 +23,16 @@ public DeleteSecretResolver(final EntityClient entityClient) {
   public CompletableFuture<String> get(final DataFetchingEnvironment environment) throws Exception {
     final QueryContext context = environment.getContext();
     if (IngestionAuthUtils.canManageSecrets(context)) {
-      final String secretUrn = environment.getArgument("urn");
-      final Urn urn = Urn.createFromString(secretUrn);
+      final String inputUrn = environment.getArgument("urn");
+      final Urn urn = Urn.createFromString(inputUrn);
       return GraphQLConcurrencyUtils.supplyAsync(
           () -> {
             try {
               _entityClient.deleteEntity(context.getOperationContext(), urn);
-              return secretUrn;
+              return inputUrn;
             } catch (Exception e) {
               throw new RuntimeException(
-                  String.format("Failed to perform delete against secret with urn %s", secretUrn),
+                  String.format("Failed to perform delete against secret with urn %s", inputUrn),
                   e);
             }
           },

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java

@@ -25,7 +25,7 @@ static String encrypt(String value, String secret) {
       } catch (NoSuchAlgorithmException e) {
         e.printStackTrace();
       }
-      Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
+      Cipher cipher = Cipher.getInstance("AES");
       cipher.init(Cipher.ENCRYPT_MODE, secretKey);
       return Base64.getEncoder()
           .encodeToString(cipher.doFinal(value.getBytes(StandardCharsets.UTF_8)));
@@ -48,7 +48,7 @@ static String decrypt(String encryptedValue, String secret) {
       } catch (NoSuchAlgorithmException e) {
         e.printStackTrace();
       }
-      Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING");
+      Cipher cipher = Cipher.getInstance("AES");
       cipher.init(Cipher.DECRYPT_MODE, secretKey);
       return new String(cipher.doFinal(Base64.getDecoder().decode(encryptedValue)));
     } catch (Exception e) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java

@@ -1,6 +1,7 @@
 package com.linkedin.datahub.graphql.resolvers.mutate;
 
 import static com.linkedin.metadata.Constants.*;
+import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata;
 
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.RecordTemplate;
@@ -84,7 +85,7 @@ private static MetadataChangeProposal setProposalProperties(
     proposal.setChangeType(ChangeType.UPSERT);
 
     // Assumes proposal is generated first from the builder methods above so SystemMetadata is empty
-    SystemMetadata systemMetadata = new SystemMetadata();
+    SystemMetadata systemMetadata = createDefaultSystemMetadata();
     StringMap properties = new StringMap();
     properties.put(APP_SOURCE, UI_SOURCE);
     systemMetadata.setProperties(properties);

datahub-graphql-core/src/main/resources/entity.graphql

@@ -68,6 +68,17 @@ type Query {
     Fetch a Tag by primary key (urn)
     """
     tag(urn: String!): Tag
+
+    """
+    Fetch a View by primary key (urn)
+    """
+    view(urn: String!): DataHubView
+
+    """
+    Fetch a Form by primary key (urn)
+    """
+    form(urn: String!): Form
+
     """
     Fetch a Role by primary key (urn)
     """

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java

@@ -12,14 +12,19 @@
 import com.datahub.plugins.auth.authorization.Authorizer;
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
 import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
 import com.linkedin.mxe.MetadataChangeProposal;
+import com.linkedin.r2.RemoteInvocationException;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.util.List;
+import java.util.stream.Collectors;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
+import org.testng.Assert;
 
 public class TestUtils {
 
@@ -120,53 +125,89 @@ public static QueryContext getMockDenyContext(String actorUrn, AuthorizationRequ
   }
 
   public static void verifyIngestProposal(
-      EntityService<ChangeItemImpl> mockService,
-      int numberOfInvocations,
-      MetadataChangeProposal proposal) {
+      EntityService<?> mockService, int numberOfInvocations, MetadataChangeProposal proposal) {
     verifyIngestProposal(mockService, numberOfInvocations, List.of(proposal));
   }
 
   public static void verifyIngestProposal(
-      EntityService<ChangeItemImpl> mockService,
+      EntityService<?> mockService,
       int numberOfInvocations,
       List<MetadataChangeProposal> proposals) {
 
-    AspectsBatchImpl batch =
-        AspectsBatchImpl.builder()
-            .mcps(
-                proposals,
-                mock(AuditStamp.class),
-                TestOperationContexts.emptyRetrieverContext(null))
-            .build();
+    ArgumentCaptor<AspectsBatchImpl> batchCaptor = ArgumentCaptor.forClass(AspectsBatchImpl.class);
+
     Mockito.verify(mockService, Mockito.times(numberOfInvocations))
-        .ingestProposal(any(), Mockito.eq(batch), Mockito.eq(false));
+        .ingestProposal(any(), batchCaptor.capture(), Mockito.eq(false));
+
+    // check has time
+    Assert.assertTrue(
+        batchCaptor.getValue().getItems().stream()
+            .allMatch(prop -> prop.getSystemMetadata().getLastObserved() > 0L));
+
+    // check without time
+    Assert.assertEquals(
+        batchCaptor.getValue().getItems().stream()
+            .map(m -> m.getSystemMetadata().setLastObserved(0))
+            .collect(Collectors.toList()),
+        proposals.stream()
+            .map(m -> m.getSystemMetadata().setLastObserved(0))
+            .collect(Collectors.toList()));
   }
 
   public static void verifySingleIngestProposal(
-      EntityService<ChangeItemImpl> mockService,
+      EntityService<?> mockService,
       int numberOfInvocations,
-      MetadataChangeProposal proposal) {
+      MetadataChangeProposal expectedProposal) {
+    ArgumentCaptor<MetadataChangeProposal> proposalCaptor =
+        ArgumentCaptor.forClass(MetadataChangeProposal.class);
+
     Mockito.verify(mockService, Mockito.times(numberOfInvocations))
-        .ingestProposal(any(), Mockito.eq(proposal), any(AuditStamp.class), Mockito.eq(false));
+        .ingestProposal(any(), proposalCaptor.capture(), any(AuditStamp.class), Mockito.eq(false));
+
+    // check has time
+    Assert.assertTrue(proposalCaptor.getValue().getSystemMetadata().getLastObserved() > 0L);
+
+    // check without time
+    proposalCaptor.getValue().getSystemMetadata().setLastObserved(0L);
+    expectedProposal.getSystemMetadata().setLastObserved(0L);
+    Assert.assertEquals(proposalCaptor.getValue(), expectedProposal);
   }
 
-  public static void verifyIngestProposal(
-      EntityService<ChangeItemImpl> mockService, int numberOfInvocations) {
+  public static void verifyIngestProposal(EntityService<?> mockService, int numberOfInvocations) {
     Mockito.verify(mockService, Mockito.times(numberOfInvocations))
         .ingestProposal(any(), any(AspectsBatchImpl.class), Mockito.eq(false));
   }
 
   public static void verifySingleIngestProposal(
-      EntityService<ChangeItemImpl> mockService, int numberOfInvocations) {
+      EntityService<?> mockService, int numberOfInvocations) {
     Mockito.verify(mockService, Mockito.times(numberOfInvocations))
         .ingestProposal(
             any(), any(MetadataChangeProposal.class), any(AuditStamp.class), Mockito.eq(false));
   }
 
-  public static void verifyNoIngestProposal(EntityService<ChangeItemImpl> mockService) {
+  public static void verifyNoIngestProposal(EntityService<?> mockService) {
     Mockito.verify(mockService, Mockito.times(0))
         .ingestProposal(any(), any(AspectsBatchImpl.class), Mockito.anyBoolean());
   }
 
+  public static void verifyIngestProposal(
+      EntityClient mockClient, int numberOfInvocations, MetadataChangeProposal expectedProposal)
+      throws RemoteInvocationException {
+
+    ArgumentCaptor<MetadataChangeProposal> proposalCaptor =
+        ArgumentCaptor.forClass(MetadataChangeProposal.class);
+
+    Mockito.verify(mockClient, Mockito.times(numberOfInvocations))
+        .ingestProposal(any(), proposalCaptor.capture(), Mockito.eq(false));
+
+    // check has time
+    Assert.assertTrue(proposalCaptor.getValue().getSystemMetadata().getLastObserved() > 0L);
+
+    // check without time
+    proposalCaptor.getValue().getSystemMetadata().setLastObserved(0L);
+    expectedProposal.getSystemMetadata().setLastObserved(0L);
+    Assert.assertEquals(proposalCaptor.getValue(), expectedProposal);
+  }
+
   private TestUtils() {}
 }

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolverTest.java

@@ -31,7 +31,7 @@ public class DeleteAssertionResolverTest {
   public void testGetSuccess() throws Exception {
     EntityClient mockClient = Mockito.mock(EntityClient.class);
 
-    EntityService mockService = getMockEntityService();
+    EntityService<?> mockService = getMockEntityService();
     Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true)))
         .thenReturn(true);
     Mockito.when(
@@ -77,7 +77,7 @@ public void testGetSuccess() throws Exception {
   public void testGetSuccessNoAssertionInfoFound() throws Exception {
     EntityClient mockClient = Mockito.mock(EntityClient.class);
 
-    EntityService mockService = getMockEntityService();
+    EntityService<?> mockService = getMockEntityService();
     Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true)))
         .thenReturn(true);
     Mockito.when(
@@ -117,7 +117,7 @@ public void testGetSuccessAssertionAlreadyRemoved() throws Exception {
     // Create resolver
     EntityClient mockClient = Mockito.mock(EntityClient.class);
 
-    EntityService mockService = getMockEntityService();
+    EntityService<?> mockService = getMockEntityService();
     Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true)))
         .thenReturn(false);
 
@@ -149,7 +149,7 @@ public void testGetSuccessAssertionAlreadyRemoved() throws Exception {
   public void testGetUnauthorized() throws Exception {
     // Create resolver
     EntityClient mockClient = Mockito.mock(EntityClient.class);
-    EntityService mockService = getMockEntityService();
+    EntityService<?> mockService = getMockEntityService();
     Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true)))
         .thenReturn(true);
     Mockito.when(
@@ -186,7 +186,7 @@ public void testGetEntityClientException() throws Exception {
         .when(mockClient)
         .deleteEntity(any(), Mockito.any());
 
-    EntityService mockService = getMockEntityService();
+    EntityService<?> mockService = getMockEntityService();
     Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true)))
         .thenReturn(true);
 

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolverTest.java

@@ -29,7 +29,7 @@ public class AddBusinessAttributeResolverTest {
       "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658";
   private static final String RESOURCE_URN =
       "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_bar)";
-  private EntityService mockService;
+  private EntityService<?> mockService;
   private QueryContext mockContext;
   private DataFetchingEnvironment mockEnv;
 

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolverTest.java

@@ -56,7 +56,7 @@ public class CreateBusinessAttributeResolverTest {
           TEST_BUSINESS_ATTRIBUTE_DESCRIPTION,
           SchemaFieldDataType.BOOLEAN);
   private EntityClient mockClient;
-  private EntityService mockService;
+  private EntityService<?> mockService;
   private QueryContext mockContext;
   private DataFetchingEnvironment mockEnv;
   private BusinessAttributeService businessAttributeService;

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolverTest.java

@@ -31,7 +31,7 @@ public class RemoveBusinessAttributeResolverTest {
       "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658";
   private static final String RESOURCE_URN =
       "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_bar)";
-  private EntityService mockService;
+  private EntityService<?> mockService;
   private QueryContext mockContext;
   private DataFetchingEnvironment mockEnv;