diff --git a/.github/scripts/docker_helpers.sh b/.github/scripts/docker_helpers.sh index 0487c69eee0ef4..e031a6d2a4d843 100755 --- a/.github/scripts/docker_helpers.sh +++ b/.github/scripts/docker_helpers.sh @@ -16,11 +16,11 @@ function get_tag { } function get_tag_slim { - echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim + echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim } function get_tag_full { - echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full + echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full } function get_python_docker_release_v { @@ -32,9 +32,9 @@ function get_unique_tag { } function get_unique_tag_slim { - echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g') + echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g') } function get_unique_tag_full { - echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g') + echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g') } diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 77874abedaabd0..52f83b3be5283d 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -30,12 +30,10 @@ jobs: "treff7es", "yoonhyejin", "eboneil", - "ethan-cartwright", "gabe-lyons", "hsheth2", "jjoyce0510", "maggiehays", - "mrjefflewis", "pedro93", "RyanHolstien", "Kunal-kankriya", @@ -45,7 +43,8 @@ jobs: "kushagra-apptware", "Salman-Apptware", "mayurinehate", - "noggi" + "noggi", + "skrydal" ]'), github.actor ) @@ -60,7 +59,6 @@ jobs: ${{ contains( fromJson('[ - "skrydal", "siladitya2", "sgomezvillamor", "ngamanda", diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java index 5e30bf976b8196..750886570bf406 100644 --- a/datahub-frontend/app/controllers/SsoCallbackController.java +++ b/datahub-frontend/app/controllers/SsoCallbackController.java @@ -66,7 +66,8 @@ public SsoCallbackController( public CompletionStage handleCallback(String protocol, Http.Request request) { if (shouldHandleCallback(protocol)) { - log.debug(String.format("Handling SSO callback. Protocol: %s", protocol)); + log.debug("Handling SSO callback. Protocol: {}", + _ssoManager.getSsoProvider().protocol().getCommonName()); return callback(request) .handle( (res, e) -> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index af5d6bc81d6146..db9bf304a1085b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -1032,6 +1032,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("mlModel", getResolver(mlModelType)) .dataFetcher("mlModelGroup", getResolver(mlModelGroupType)) .dataFetcher("assertion", getResolver(assertionType)) + .dataFetcher("form", getResolver(formType)) + .dataFetcher("view", getResolver(dataHubViewType)) .dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient)) .dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver()) .dataFetcher("listUsers", new ListUsersResolver(this.entityClient)) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java index 53ae6d4509e7df..eb152087699024 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java @@ -41,7 +41,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw final QueryContext context = environment.getContext(); final String tokenId = bindArgument(environment.getArgument("tokenId"), String.class); - log.info("User {} revoking access token {}", context.getActorUrn(), tokenId); + log.info("User {} revoking access token", context.getActorUrn()); if (isAuthorizedToRevokeToken(context, tokenId)) { try { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java index f557b9889f604d..da81d560c6dbd6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java @@ -23,16 +23,16 @@ public DeleteSecretResolver(final EntityClient entityClient) { public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final QueryContext context = environment.getContext(); if (IngestionAuthUtils.canManageSecrets(context)) { - final String secretUrn = environment.getArgument("urn"); - final Urn urn = Urn.createFromString(secretUrn); + final String inputUrn = environment.getArgument("urn"); + final Urn urn = Urn.createFromString(inputUrn); return GraphQLConcurrencyUtils.supplyAsync( () -> { try { _entityClient.deleteEntity(context.getOperationContext(), urn); - return secretUrn; + return inputUrn; } catch (Exception e) { throw new RuntimeException( - String.format("Failed to perform delete against secret with urn %s", secretUrn), + String.format("Failed to perform delete against secret with urn %s", inputUrn), e); } }, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java index 225a5801adec94..87a3e5cb79ebfc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java @@ -25,7 +25,7 @@ static String encrypt(String value, String secret) { } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } - Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding"); + Cipher cipher = Cipher.getInstance("AES"); cipher.init(Cipher.ENCRYPT_MODE, secretKey); return Base64.getEncoder() .encodeToString(cipher.doFinal(value.getBytes(StandardCharsets.UTF_8))); @@ -48,7 +48,7 @@ static String decrypt(String encryptedValue, String secret) { } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } - Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING"); + Cipher cipher = Cipher.getInstance("AES"); cipher.init(Cipher.DECRYPT_MODE, secretKey); return new String(cipher.doFinal(Base64.getDecoder().decode(encryptedValue))); } catch (Exception e) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java index 8b798b243ca864..7608007e9dda98 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.resolvers.mutate; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; @@ -84,7 +85,7 @@ private static MetadataChangeProposal setProposalProperties( proposal.setChangeType(ChangeType.UPSERT); // Assumes proposal is generated first from the builder methods above so SystemMetadata is empty - SystemMetadata systemMetadata = new SystemMetadata(); + SystemMetadata systemMetadata = createDefaultSystemMetadata(); StringMap properties = new StringMap(); properties.put(APP_SOURCE, UI_SOURCE); systemMetadata.setProperties(properties); diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index a2440f7e3928ee..941a6a28ceb2c7 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -68,6 +68,17 @@ type Query { Fetch a Tag by primary key (urn) """ tag(urn: String!): Tag + + """ + Fetch a View by primary key (urn) + """ + view(urn: String!): DataHubView + + """ + Fetch a Form by primary key (urn) + """ + form(urn: String!): Form + """ Fetch a Role by primary key (urn) """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java index 0df5d162bab442..522e4be0ec5ec2 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java @@ -12,14 +12,19 @@ import com.datahub.plugins.auth.authorization.Authorizer; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; import java.util.List; +import java.util.stream.Collectors; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import org.testng.Assert; public class TestUtils { @@ -120,53 +125,89 @@ public static QueryContext getMockDenyContext(String actorUrn, AuthorizationRequ } public static void verifyIngestProposal( - EntityService mockService, - int numberOfInvocations, - MetadataChangeProposal proposal) { + EntityService mockService, int numberOfInvocations, MetadataChangeProposal proposal) { verifyIngestProposal(mockService, numberOfInvocations, List.of(proposal)); } public static void verifyIngestProposal( - EntityService mockService, + EntityService mockService, int numberOfInvocations, List proposals) { - AspectsBatchImpl batch = - AspectsBatchImpl.builder() - .mcps( - proposals, - mock(AuditStamp.class), - TestOperationContexts.emptyRetrieverContext(null)) - .build(); + ArgumentCaptor batchCaptor = ArgumentCaptor.forClass(AspectsBatchImpl.class); + Mockito.verify(mockService, Mockito.times(numberOfInvocations)) - .ingestProposal(any(), Mockito.eq(batch), Mockito.eq(false)); + .ingestProposal(any(), batchCaptor.capture(), Mockito.eq(false)); + + // check has time + Assert.assertTrue( + batchCaptor.getValue().getItems().stream() + .allMatch(prop -> prop.getSystemMetadata().getLastObserved() > 0L)); + + // check without time + Assert.assertEquals( + batchCaptor.getValue().getItems().stream() + .map(m -> m.getSystemMetadata().setLastObserved(0)) + .collect(Collectors.toList()), + proposals.stream() + .map(m -> m.getSystemMetadata().setLastObserved(0)) + .collect(Collectors.toList())); } public static void verifySingleIngestProposal( - EntityService mockService, + EntityService mockService, int numberOfInvocations, - MetadataChangeProposal proposal) { + MetadataChangeProposal expectedProposal) { + ArgumentCaptor proposalCaptor = + ArgumentCaptor.forClass(MetadataChangeProposal.class); + Mockito.verify(mockService, Mockito.times(numberOfInvocations)) - .ingestProposal(any(), Mockito.eq(proposal), any(AuditStamp.class), Mockito.eq(false)); + .ingestProposal(any(), proposalCaptor.capture(), any(AuditStamp.class), Mockito.eq(false)); + + // check has time + Assert.assertTrue(proposalCaptor.getValue().getSystemMetadata().getLastObserved() > 0L); + + // check without time + proposalCaptor.getValue().getSystemMetadata().setLastObserved(0L); + expectedProposal.getSystemMetadata().setLastObserved(0L); + Assert.assertEquals(proposalCaptor.getValue(), expectedProposal); } - public static void verifyIngestProposal( - EntityService mockService, int numberOfInvocations) { + public static void verifyIngestProposal(EntityService mockService, int numberOfInvocations) { Mockito.verify(mockService, Mockito.times(numberOfInvocations)) .ingestProposal(any(), any(AspectsBatchImpl.class), Mockito.eq(false)); } public static void verifySingleIngestProposal( - EntityService mockService, int numberOfInvocations) { + EntityService mockService, int numberOfInvocations) { Mockito.verify(mockService, Mockito.times(numberOfInvocations)) .ingestProposal( any(), any(MetadataChangeProposal.class), any(AuditStamp.class), Mockito.eq(false)); } - public static void verifyNoIngestProposal(EntityService mockService) { + public static void verifyNoIngestProposal(EntityService mockService) { Mockito.verify(mockService, Mockito.times(0)) .ingestProposal(any(), any(AspectsBatchImpl.class), Mockito.anyBoolean()); } + public static void verifyIngestProposal( + EntityClient mockClient, int numberOfInvocations, MetadataChangeProposal expectedProposal) + throws RemoteInvocationException { + + ArgumentCaptor proposalCaptor = + ArgumentCaptor.forClass(MetadataChangeProposal.class); + + Mockito.verify(mockClient, Mockito.times(numberOfInvocations)) + .ingestProposal(any(), proposalCaptor.capture(), Mockito.eq(false)); + + // check has time + Assert.assertTrue(proposalCaptor.getValue().getSystemMetadata().getLastObserved() > 0L); + + // check without time + proposalCaptor.getValue().getSystemMetadata().setLastObserved(0L); + expectedProposal.getSystemMetadata().setLastObserved(0L); + Assert.assertEquals(proposalCaptor.getValue(), expectedProposal); + } + private TestUtils() {} } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolverTest.java index 4dd09c1e5cfd5f..948088175e8e63 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolverTest.java @@ -31,7 +31,7 @@ public class DeleteAssertionResolverTest { public void testGetSuccess() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true))) .thenReturn(true); Mockito.when( @@ -77,7 +77,7 @@ public void testGetSuccess() throws Exception { public void testGetSuccessNoAssertionInfoFound() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true))) .thenReturn(true); Mockito.when( @@ -117,7 +117,7 @@ public void testGetSuccessAssertionAlreadyRemoved() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true))) .thenReturn(false); @@ -149,7 +149,7 @@ public void testGetSuccessAssertionAlreadyRemoved() throws Exception { public void testGetUnauthorized() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true))) .thenReturn(true); Mockito.when( @@ -186,7 +186,7 @@ public void testGetEntityClientException() throws Exception { .when(mockClient) .deleteEntity(any(), Mockito.any()); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ASSERTION_URN)), eq(true))) .thenReturn(true); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolverTest.java index 1a0e558e309d7b..280adcf896d5e9 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolverTest.java @@ -29,7 +29,7 @@ public class AddBusinessAttributeResolverTest { "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658"; private static final String RESOURCE_URN = "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_bar)"; - private EntityService mockService; + private EntityService mockService; private QueryContext mockContext; private DataFetchingEnvironment mockEnv; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolverTest.java index 574b81bb86630f..2623a6b25811ad 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolverTest.java @@ -56,7 +56,7 @@ public class CreateBusinessAttributeResolverTest { TEST_BUSINESS_ATTRIBUTE_DESCRIPTION, SchemaFieldDataType.BOOLEAN); private EntityClient mockClient; - private EntityService mockService; + private EntityService mockService; private QueryContext mockContext; private DataFetchingEnvironment mockEnv; private BusinessAttributeService businessAttributeService; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolverTest.java index 32a12d3ee8607e..3e7df667160624 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolverTest.java @@ -31,7 +31,7 @@ public class RemoveBusinessAttributeResolverTest { "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658"; private static final String RESOURCE_URN = "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_bar)"; - private EntityService mockService; + private EntityService mockService; private QueryContext mockContext; private DataFetchingEnvironment mockEnv; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolverTest.java index 601fc56b251495..bf01b54c7ed726 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolverTest.java @@ -2,6 +2,7 @@ import static com.linkedin.datahub.graphql.TestUtils.*; import static com.linkedin.datahub.graphql.resolvers.datacontract.EntityDataContractResolver.*; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; import static org.mockito.ArgumentMatchers.any; import static org.testng.Assert.*; @@ -43,14 +44,19 @@ import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; -import com.linkedin.mxe.SystemMetadata; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; import io.datahubproject.metadata.context.OperationContext; import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; import org.testng.Assert; +import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; public class UpsertDataContractResolverTest { @@ -83,9 +89,15 @@ public class UpsertDataContractResolverTest { private static final Urn TEST_ACTOR_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + @Captor private ArgumentCaptor> proposalCaptor; + + @BeforeTest + public void init() { + MockitoAnnotations.openMocks(this); + } + @Test public void testGetSuccessCreate() throws Exception { - // Expected results final DataContractKey key = new DataContractKey(); key.setId("test-id"); @@ -127,7 +139,8 @@ public void testGetSuccessCreate() throws Exception { propertiesProposal.setEntityUrn(dataContractUrn); propertiesProposal.setEntityType(Constants.DATA_CONTRACT_ENTITY_NAME); propertiesProposal.setSystemMetadata( - new SystemMetadata().setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); + createDefaultSystemMetadata() + .setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); propertiesProposal.setAspectName(Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME); propertiesProposal.setAspect(GenericRecordUtils.serializeAspect(props)); propertiesProposal.setChangeType(ChangeType.UPSERT); @@ -136,16 +149,29 @@ public void testGetSuccessCreate() throws Exception { statusProposal.setEntityUrn(dataContractUrn); statusProposal.setEntityType(Constants.DATA_CONTRACT_ENTITY_NAME); statusProposal.setSystemMetadata( - new SystemMetadata().setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); + createDefaultSystemMetadata() + .setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); statusProposal.setAspectName(Constants.DATA_CONTRACT_STATUS_ASPECT_NAME); statusProposal.setAspect(GenericRecordUtils.serializeAspect(status)); statusProposal.setChangeType(ChangeType.UPSERT); Mockito.verify(mockClient, Mockito.times(1)) .batchIngestProposals( - any(OperationContext.class), - Mockito.eq(ImmutableList.of(propertiesProposal, statusProposal)), - Mockito.eq(false)); + any(OperationContext.class), proposalCaptor.capture(), Mockito.eq(false)); + + // check has time + Assert.assertTrue( + proposalCaptor.getValue().stream() + .allMatch(prop -> prop.getSystemMetadata().getLastObserved() > 0L)); + + // check without time + Assert.assertEquals( + proposalCaptor.getValue().stream() + .map(m -> m.getSystemMetadata().setLastObserved(0)) + .collect(Collectors.toList()), + List.of(propertiesProposal, statusProposal).stream() + .map(m -> m.getSystemMetadata().setLastObserved(0)) + .collect(Collectors.toList())); Assert.assertEquals(result.getUrn(), TEST_CONTRACT_URN.toString()); } @@ -188,7 +214,8 @@ public void testGetSuccessUpdate() throws Exception { propertiesProposal.setEntityUrn(TEST_CONTRACT_URN); propertiesProposal.setEntityType(Constants.DATA_CONTRACT_ENTITY_NAME); propertiesProposal.setSystemMetadata( - new SystemMetadata().setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); + createDefaultSystemMetadata() + .setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); propertiesProposal.setAspectName(Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME); propertiesProposal.setAspect(GenericRecordUtils.serializeAspect(props)); propertiesProposal.setChangeType(ChangeType.UPSERT); @@ -197,16 +224,29 @@ public void testGetSuccessUpdate() throws Exception { statusProposal.setEntityUrn(TEST_CONTRACT_URN); statusProposal.setEntityType(Constants.DATA_CONTRACT_ENTITY_NAME); statusProposal.setSystemMetadata( - new SystemMetadata().setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); + createDefaultSystemMetadata() + .setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); statusProposal.setAspectName(Constants.DATA_CONTRACT_STATUS_ASPECT_NAME); statusProposal.setAspect(GenericRecordUtils.serializeAspect(status)); statusProposal.setChangeType(ChangeType.UPSERT); Mockito.verify(mockClient, Mockito.times(1)) .batchIngestProposals( - any(OperationContext.class), - Mockito.eq(ImmutableList.of(propertiesProposal, statusProposal)), - Mockito.eq(false)); + any(OperationContext.class), proposalCaptor.capture(), Mockito.eq(false)); + + // check has time + Assert.assertTrue( + proposalCaptor.getValue().stream() + .allMatch(prop -> prop.getSystemMetadata().getLastObserved() > 0L)); + + // check without time + Assert.assertEquals( + proposalCaptor.getValue().stream() + .map(m -> m.getSystemMetadata().setLastObserved(0)) + .collect(Collectors.toList()), + List.of(propertiesProposal, statusProposal).stream() + .map(m -> m.getSystemMetadata().setLastObserved(0)) + .collect(Collectors.toList())); Assert.assertEquals(result.getUrn(), TEST_CONTRACT_URN.toString()); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java index 02c9212e7f563d..48b3dc5f6db943 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java @@ -33,7 +33,7 @@ public class BatchUpdateSoftDeletedResolverTest { @Test public void testGetSuccessNoExistingStatus() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -84,7 +84,7 @@ public void testGetSuccessNoExistingStatus() throws Exception { public void testGetSuccessExistingStatus() throws Exception { final Status originalStatus = new Status().setRemoved(true); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -133,7 +133,7 @@ public void testGetSuccessExistingStatus() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -173,7 +173,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchUpdateSoftDeletedResolver resolver = new BatchUpdateSoftDeletedResolver(mockService); @@ -193,7 +193,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java index b7324d210fc212..265a1a2e0af5ba 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java @@ -34,7 +34,7 @@ public class BatchUpdateDeprecationResolverTest { @Test public void testGetSuccessNoExistingDeprecation() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -99,7 +99,7 @@ public void testGetSuccessExistingDeprecation() throws Exception { .setNote("") .setActor(UrnUtils.getUrn("urn:li:corpuser:test")); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -158,7 +158,7 @@ public void testGetSuccessExistingDeprecation() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -202,7 +202,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchUpdateDeprecationResolver resolver = new BatchUpdateDeprecationResolver(mockService); @@ -226,7 +226,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolverTest.java index 09894ccf49f227..ab180724da46df 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolverTest.java @@ -59,7 +59,7 @@ public void testGetSuccessNoExistingDeprecation() throws Exception { .setUrn(Urn.createFromString(TEST_ENTITY_URN)) .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(true); @@ -83,7 +83,7 @@ public void testGetSuccessNoExistingDeprecation() throws Exception { MutationUtils.buildMetadataChangeProposalWithUrn( UrnUtils.getUrn(TEST_ENTITY_URN), DEPRECATION_ASPECT_NAME, newDeprecation); - Mockito.verify(mockClient, Mockito.times(1)).ingestProposal(any(), eq(proposal), eq(false)); + verifyIngestProposal(mockClient, 1, proposal); Mockito.verify(mockService, Mockito.times(1)) .exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true)); @@ -120,7 +120,7 @@ public void testGetSuccessExistingDeprecation() throws Exception { new EnvelopedAspect() .setValue(new Aspect(originalDeprecation.data()))))))); - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(true); @@ -144,7 +144,7 @@ public void testGetSuccessExistingDeprecation() throws Exception { MutationUtils.buildMetadataChangeProposalWithUrn( UrnUtils.getUrn(TEST_ENTITY_URN), DEPRECATION_ASPECT_NAME, newDeprecation); - Mockito.verify(mockClient, Mockito.times(1)).ingestProposal(any(), eq(proposal), eq(false)); + verifyIngestProposal(mockClient, 1, proposal); Mockito.verify(mockService, Mockito.times(1)) .exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true)); @@ -169,7 +169,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { .setUrn(Urn.createFromString(TEST_ENTITY_URN)) .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(false); @@ -190,7 +190,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { public void testGetUnauthorized() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); UpdateDeprecationResolver resolver = new UpdateDeprecationResolver(mockClient, mockService); // Execute resolver @@ -206,7 +206,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); Mockito.doThrow(RemoteInvocationException.class) .when(mockClient) .ingestProposal(any(), Mockito.any(), anyBoolean()); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java index 9fec8b2fd9572c..1a9272c1335cf9 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java @@ -39,7 +39,7 @@ public class BatchSetDomainResolverTest { @Test public void testGetSuccessNoExistingDomains() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -105,7 +105,7 @@ public void testGetSuccessExistingDomains() throws Exception { new Domains() .setDomains(new UrnArray(ImmutableList.of(Urn.createFromString(TEST_DOMAIN_1_URN)))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -176,7 +176,7 @@ public void testGetSuccessUnsetDomains() throws Exception { new Domains() .setDomains(new UrnArray(ImmutableList.of(Urn.createFromString(TEST_DOMAIN_1_URN)))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -233,7 +233,7 @@ public void testGetSuccessUnsetDomains() throws Exception { @Test public void testGetFailureDomainDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -268,7 +268,7 @@ public void testGetFailureDomainDoesNotExist() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -312,7 +312,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchSetDomainResolver resolver = new BatchSetDomainResolver(mockService); @@ -334,7 +334,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index 1d6c4519358b45..c0d74225a9cf1d 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -53,7 +53,7 @@ public class CreateDomainResolverTest { public void testGetSuccess() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); Mockito.when(mockClient.exists(any(), Mockito.eq(TEST_DOMAIN_URN))).thenReturn(false); @@ -103,7 +103,7 @@ public void testGetSuccess() throws Exception { @Test public void testGetSuccessNoParentDomain() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); Mockito.when(mockClient.exists(any(), Mockito.eq(TEST_DOMAIN_URN))).thenReturn(false); @@ -146,7 +146,7 @@ public void testGetSuccessNoParentDomain() throws Exception { @Test public void testGetInvalidParent() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); Mockito.when(mockClient.exists(any(), Mockito.eq(TEST_DOMAIN_URN))).thenReturn(false); @@ -164,7 +164,7 @@ public void testGetInvalidParent() throws Exception { @Test public void testGetNameConflict() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); Mockito.when(mockClient.exists(any(), Mockito.eq(TEST_DOMAIN_URN))).thenReturn(false); @@ -218,7 +218,7 @@ public void testGetNameConflict() throws Exception { public void testGetUnauthorized() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); // Execute resolver @@ -235,7 +235,7 @@ public void testGetUnauthorized() throws Exception { public void testGetEntityClientException() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RemoteInvocationException.class) .when(mockClient) .ingestProposal(any(), Mockito.any(), Mockito.eq(false)); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java index ae1dffbd5d0db8..07fad314747db8 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java @@ -40,7 +40,7 @@ public class MoveDomainResolverTest { private static final CorpuserUrn TEST_ACTOR_URN = new CorpuserUrn("test"); private MetadataChangeProposal setupTests( - DataFetchingEnvironment mockEnv, EntityService mockService, EntityClient mockClient) + DataFetchingEnvironment mockEnv, EntityService mockService, EntityClient mockClient) throws Exception { QueryContext mockContext = getMockAllowContext(); Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); @@ -77,7 +77,7 @@ private MetadataChangeProposal setupTests( @Test public void testGetSuccess() throws Exception { - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when( mockService.exists( @@ -100,7 +100,7 @@ public void testGetSuccess() throws Exception { @Test public void testGetFailureEntityDoesNotExist() throws Exception { - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(PARENT_DOMAIN_URN)), eq(true))) .thenReturn(true); @@ -127,7 +127,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { @Test public void testGetFailureParentDoesNotExist() throws Exception { - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(PARENT_DOMAIN_URN)), eq(true))) .thenReturn(false); @@ -143,7 +143,7 @@ public void testGetFailureParentDoesNotExist() throws Exception { @Test public void testGetFailureParentIsNotDomain() throws Exception { - EntityService mockService = Mockito.mock(EntityService.class); + EntityService mockService = Mockito.mock(EntityService.class); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(PARENT_DOMAIN_URN)), eq(true))) .thenReturn(true); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java index 031ac1da8480bf..5437f1c860fde6 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java @@ -59,7 +59,7 @@ public void testGetSuccessNoExistingDomains() throws Exception { .setUrn(Urn.createFromString(TEST_ENTITY_URN)) .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(true); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_NEW_DOMAIN_URN)), eq(true))) @@ -82,8 +82,7 @@ public void testGetSuccessNoExistingDomains() throws Exception { MutationUtils.buildMetadataChangeProposalWithUrn( UrnUtils.getUrn(TEST_ENTITY_URN), DOMAINS_ASPECT_NAME, newDomains); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); Mockito.verify(mockService, Mockito.times(1)) .exists(any(), Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true)); @@ -122,7 +121,7 @@ public void testGetSuccessExistingDomains() throws Exception { new EnvelopedAspect() .setValue(new Aspect(originalDomains.data()))))))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(true); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_NEW_DOMAIN_URN)), eq(true))) @@ -145,8 +144,7 @@ public void testGetSuccessExistingDomains() throws Exception { MutationUtils.buildMetadataChangeProposalWithUrn( UrnUtils.getUrn(TEST_ENTITY_URN), DOMAINS_ASPECT_NAME, newDomains); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); Mockito.verify(mockService, Mockito.times(1)) .exists(any(), Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true)); @@ -175,7 +173,7 @@ public void testGetFailureDomainDoesNotExist() throws Exception { .setUrn(Urn.createFromString(TEST_ENTITY_URN)) .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(true); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_NEW_DOMAIN_URN)), eq(true))) @@ -214,7 +212,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { .setUrn(Urn.createFromString(TEST_ENTITY_URN)) .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(false); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_NEW_DOMAIN_URN)), eq(true))) @@ -237,7 +235,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { public void testGetUnauthorized() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); SetDomainResolver resolver = new SetDomainResolver(mockClient, mockService); // Execute resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolverTest.java index b9910e6ca3c85e..1c61963703a2ab 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolverTest.java @@ -58,7 +58,7 @@ public void testGetSuccessNoExistingDomains() throws Exception { .setUrn(Urn.createFromString(TEST_ENTITY_URN)) .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(true); @@ -76,8 +76,7 @@ public void testGetSuccessNoExistingDomains() throws Exception { MutationUtils.buildMetadataChangeProposalWithUrn( UrnUtils.getUrn(TEST_ENTITY_URN), DOMAINS_ASPECT_NAME, newDomains); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); Mockito.verify(mockService, Mockito.times(1)) .exists(any(), Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true)); @@ -113,7 +112,7 @@ public void testGetSuccessExistingDomains() throws Exception { new EnvelopedAspect() .setValue(new Aspect(originalDomains.data()))))))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(true); @@ -131,8 +130,7 @@ public void testGetSuccessExistingDomains() throws Exception { MutationUtils.buildMetadataChangeProposalWithUrn( UrnUtils.getUrn(TEST_ENTITY_URN), DOMAINS_ASPECT_NAME, newDomains); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); Mockito.verify(mockService, Mockito.times(1)) .exists(any(), Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true)); @@ -158,7 +156,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { .setUrn(Urn.createFromString(TEST_ENTITY_URN)) .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(false); @@ -178,7 +176,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { public void testGetUnauthorized() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); UnsetDomainResolver resolver = new UnsetDomainResolver(mockClient, mockService); // Execute resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java index 2bb9dff29a5643..c45e620a46a73d 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java @@ -39,7 +39,7 @@ public class UpdateEmbedResolverTest { @Test public void testGetSuccessNoExistingEmbed() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -68,7 +68,6 @@ public void testGetSuccessNoExistingEmbed() throws Exception { Urn.createFromString(TEST_ENTITY_URN), EMBED_ASPECT_NAME, newEmbed); verifySingleIngestProposal(mockService, 1, proposal); - ; Mockito.verify(mockService, Mockito.times(1)) .exists(any(), Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true)); @@ -79,7 +78,7 @@ public void testGetSuccessExistingEmbed() throws Exception { Embed originalEmbed = new Embed().setRenderUrl("https://otherurl.com"); // Create resolver - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -132,7 +131,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { .setUrn(Urn.createFromString(TEST_ENTITY_URN)) .setAspects(new EnvelopedAspectMap(Collections.emptyMap())))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(false); @@ -154,7 +153,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { // Create resolver - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); UpdateEmbedResolver resolver = new UpdateEmbedResolver(mockService); // Execute resolver @@ -171,7 +170,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RemoteInvocationException.class) .when(mockClient) .ingestProposal(any(), Mockito.any()); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolverTest.java index ffa9e4a728dda4..a8920fa9e5b3c2 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolverTest.java @@ -28,7 +28,7 @@ public class AddRelatedTermsResolverTest { private static final String DATASET_URN = "urn:li:dataset:(test,test,test)"; private EntityService setUpService() { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( any(), @@ -41,7 +41,7 @@ private EntityService setUpService() { @Test public void testGetSuccessIsRelatedNonExistent() throws Exception { - EntityService mockService = setUpService(); + EntityService mockService = setUpService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) @@ -75,7 +75,7 @@ public void testGetSuccessIsRelatedNonExistent() throws Exception { @Test public void testGetSuccessHasRelatedNonExistent() throws Exception { - EntityService mockService = setUpService(); + EntityService mockService = setUpService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) @@ -109,7 +109,7 @@ public void testGetSuccessHasRelatedNonExistent() throws Exception { @Test public void testGetFailAddSelfAsRelatedTerm() throws Exception { - EntityService mockService = setUpService(); + EntityService mockService = setUpService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) @@ -131,7 +131,7 @@ public void testGetFailAddSelfAsRelatedTerm() throws Exception { @Test public void testGetFailAddNonTermAsRelatedTerm() throws Exception { - EntityService mockService = setUpService(); + EntityService mockService = setUpService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) @@ -153,7 +153,7 @@ public void testGetFailAddNonTermAsRelatedTerm() throws Exception { @Test public void testGetFailAddNonExistentTermAsRelatedTerm() throws Exception { - EntityService mockService = setUpService(); + EntityService mockService = setUpService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) @@ -177,7 +177,7 @@ public void testGetFailAddNonExistentTermAsRelatedTerm() throws Exception { @Test public void testGetFailAddToNonExistentUrn() throws Exception { - EntityService mockService = setUpService(); + EntityService mockService = setUpService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) @@ -201,7 +201,7 @@ public void testGetFailAddToNonExistentUrn() throws Exception { @Test public void testGetFailAddToNonTerm() throws Exception { - EntityService mockService = setUpService(); + EntityService mockService = setUpService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(DATASET_URN)), eq(true))) @@ -225,7 +225,7 @@ public void testGetFailAddToNonTerm() throws Exception { @Test public void testFailNoPermissions() throws Exception { - EntityService mockService = setUpService(); + EntityService mockService = setUpService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolverTest.java index 1a7e74c36733ca..0f2fa7f88cd9b6 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolverTest.java @@ -2,8 +2,8 @@ import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; import static com.linkedin.datahub.graphql.TestUtils.getMockEntityService; +import static com.linkedin.datahub.graphql.TestUtils.verifyIngestProposal; import static com.linkedin.metadata.Constants.*; -import static org.mockito.ArgumentMatchers.any; import com.datahub.authentication.Authentication; import com.linkedin.common.urn.GlossaryNodeUrn; @@ -63,7 +63,7 @@ private MetadataChangeProposal setupTest( @Test public void testGetSuccess() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); final MetadataChangeProposal proposal = setupTest(mockEnv, TEST_INPUT, "test-description", parentNodeUrn); @@ -71,14 +71,13 @@ public void testGetSuccess() throws Exception { CreateGlossaryNodeResolver resolver = new CreateGlossaryNodeResolver(mockClient, mockService); resolver.get(mockEnv).get(); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); } @Test public void testGetSuccessNoDescription() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); final MetadataChangeProposal proposal = setupTest(mockEnv, TEST_INPUT_NO_DESCRIPTION, "", parentNodeUrn); @@ -86,14 +85,13 @@ public void testGetSuccessNoDescription() throws Exception { CreateGlossaryNodeResolver resolver = new CreateGlossaryNodeResolver(mockClient, mockService); resolver.get(mockEnv).get(); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); } @Test public void testGetSuccessNoParentNode() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); final MetadataChangeProposal proposal = setupTest(mockEnv, TEST_INPUT_NO_PARENT_NODE, "test-description", null); @@ -101,7 +99,6 @@ public void testGetSuccessNoParentNode() throws Exception { CreateGlossaryNodeResolver resolver = new CreateGlossaryNodeResolver(mockClient, mockService); resolver.get(mockEnv).get(); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); } } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolverTest.java index 728a840d97e94e..8a51d8ea100092 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolverTest.java @@ -2,6 +2,7 @@ import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; import static com.linkedin.datahub.graphql.TestUtils.getMockEntityService; +import static com.linkedin.datahub.graphql.TestUtils.verifyIngestProposal; import static com.linkedin.metadata.Constants.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -81,7 +82,7 @@ private MetadataChangeProposal setupTest( @Test public void testGetSuccess() throws Exception { EntityClient mockClient = initMockClient(); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); final MetadataChangeProposal proposal = setupTest(mockEnv, TEST_INPUT, "test-description", parentNodeUrn); @@ -89,14 +90,13 @@ public void testGetSuccess() throws Exception { CreateGlossaryTermResolver resolver = new CreateGlossaryTermResolver(mockClient, mockService); resolver.get(mockEnv).get(); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); } @Test public void testGetSuccessNoDescription() throws Exception { EntityClient mockClient = initMockClient(); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); final MetadataChangeProposal proposal = setupTest(mockEnv, TEST_INPUT_NO_DESCRIPTION, "", parentNodeUrn); @@ -104,14 +104,13 @@ public void testGetSuccessNoDescription() throws Exception { CreateGlossaryTermResolver resolver = new CreateGlossaryTermResolver(mockClient, mockService); resolver.get(mockEnv).get(); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); } @Test public void testGetSuccessNoParentNode() throws Exception { EntityClient mockClient = initMockClient(); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); final MetadataChangeProposal proposal = setupTest(mockEnv, TEST_INPUT_NO_PARENT_NODE, "test-description", null); @@ -119,8 +118,7 @@ public void testGetSuccessNoParentNode() throws Exception { CreateGlossaryTermResolver resolver = new CreateGlossaryTermResolver(mockClient, mockService); resolver.get(mockEnv).get(); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); } @Test @@ -157,7 +155,7 @@ public void testGetFailureExistingTermSameName() throws Exception { Mockito.eq(Collections.singleton(GLOSSARY_TERM_INFO_ASPECT_NAME)))) .thenReturn(result); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); CreateGlossaryEntityInput input = diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/DeleteGlossaryEntityResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/DeleteGlossaryEntityResolverTest.java index 18c4b07ffeff04..9adc5d5e516e52 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/DeleteGlossaryEntityResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/DeleteGlossaryEntityResolverTest.java @@ -25,7 +25,7 @@ public class DeleteGlossaryEntityResolverTest { @Test public void testGetSuccess() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_TERM_URN)), eq(true))) .thenReturn(true); @@ -50,7 +50,7 @@ public void testGetEntityClientException() throws Exception { .when(mockClient) .deleteEntity(any(), Mockito.any()); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_TERM_URN)), eq(true))) .thenReturn(true); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolverTest.java index 5de7966dc15e99..85019a475865e1 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolverTest.java @@ -36,7 +36,7 @@ public void testGetSuccessIsA() throws Exception { GlossaryTermUrn term2Urn = GlossaryTermUrn.createFromString(TEST_TERM_2_URN); final GlossaryRelatedTerms relatedTerms = new GlossaryRelatedTerms(); relatedTerms.setIsRelatedTerms(new GlossaryTermUrnArray(Arrays.asList(term1Urn, term2Urn))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( any(), @@ -71,7 +71,7 @@ public void testGetSuccessHasA() throws Exception { GlossaryTermUrn term2Urn = GlossaryTermUrn.createFromString(TEST_TERM_2_URN); final GlossaryRelatedTerms relatedTerms = new GlossaryRelatedTerms(); relatedTerms.setHasRelatedTerms(new GlossaryTermUrnArray(Arrays.asList(term1Urn, term2Urn))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( any(), @@ -102,7 +102,7 @@ public void testGetSuccessHasA() throws Exception { @Test public void testFailAspectDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( any(), @@ -135,7 +135,7 @@ public void testFailNoPermissions() throws Exception { GlossaryTermUrn term2Urn = GlossaryTermUrn.createFromString(TEST_TERM_2_URN); final GlossaryRelatedTerms relatedTerms = new GlossaryRelatedTerms(); relatedTerms.setIsRelatedTerms(new GlossaryTermUrnArray(Arrays.asList(term1Urn, term2Urn))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( any(), diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java index e6f4aae56127ce..b4a2655755a028 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java @@ -42,7 +42,7 @@ public class UpdateNameResolverTest { private static final CorpuserUrn TEST_ACTOR_URN = new CorpuserUrn("test"); private MetadataChangeProposal setupTests( - DataFetchingEnvironment mockEnv, EntityService mockService) throws Exception { + DataFetchingEnvironment mockEnv, EntityService mockService) throws Exception { QueryContext mockContext = getMockAllowContext(); Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); @@ -65,7 +65,7 @@ private MetadataChangeProposal setupTests( @Test public void testGetSuccess() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when( mockService.exists( @@ -83,7 +83,7 @@ public void testGetSuccess() throws Exception { @Test public void testGetSuccessForNode() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(NODE_URN)), eq(true))) .thenReturn(true); @@ -117,7 +117,7 @@ public void testGetSuccessForNode() throws Exception { @Test public void testGetSuccessForDomain() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(DOMAIN_URN)), eq(true))) .thenReturn(true); @@ -163,7 +163,7 @@ public void testGetSuccessForDomain() throws Exception { @Test public void testGetFailureEntityDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when( mockService.exists( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateParentNodeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateParentNodeResolverTest.java index 39f9066bcddaa0..25a900d4d90696 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateParentNodeResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateParentNodeResolverTest.java @@ -43,7 +43,7 @@ public class UpdateParentNodeResolverTest { private static final CorpuserUrn TEST_ACTOR_URN = new CorpuserUrn("test"); private MetadataChangeProposal setupTests( - DataFetchingEnvironment mockEnv, EntityService mockService) throws Exception { + DataFetchingEnvironment mockEnv, EntityService mockService) throws Exception { QueryContext mockContext = getMockAllowContext(); Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); @@ -67,7 +67,7 @@ private MetadataChangeProposal setupTests( @Test public void testGetSuccess() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TERM_URN)), eq(true))) .thenReturn(true); @@ -89,7 +89,7 @@ public void testGetSuccess() throws Exception { @Test public void testGetSuccessForNode() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(NODE_URN)), eq(true))) .thenReturn(true); @@ -129,7 +129,7 @@ public void testGetSuccessForNode() throws Exception { @Test public void testGetFailureEntityDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TERM_URN)), eq(true))) .thenReturn(false); @@ -149,7 +149,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { @Test public void testGetFailureNodeDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TERM_URN)), eq(true))) .thenReturn(true); @@ -169,7 +169,7 @@ public void testGetFailureNodeDoesNotExist() throws Exception { @Test public void testGetFailureParentIsNotNode() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TERM_URN)), eq(true))) .thenReturn(true); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java index 57d96030a32aaa..955188a4e4fed1 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.resolvers.ingest.source; +import static com.linkedin.datahub.graphql.TestUtils.verifyIngestProposal; import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*; import static com.linkedin.metadata.Constants.*; import static org.mockito.ArgumentMatchers.any; @@ -66,13 +67,11 @@ public void testGetSuccess() throws Exception { .setExecutorId(TEST_INPUT.getConfig().getExecutorId()) .setDebugMode(TEST_INPUT.getConfig().getDebugMode())); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal( - any(), - Mockito.eq( - MutationUtils.buildMetadataChangeProposalWithUrn( - TEST_INGESTION_SOURCE_URN, INGESTION_INFO_ASPECT_NAME, info)), - Mockito.eq(false)); + verifyIngestProposal( + mockClient, + 1, + MutationUtils.buildMetadataChangeProposalWithUrn( + TEST_INGESTION_SOURCE_URN, INGESTION_INFO_ASPECT_NAME, info)); } @Test diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java index 5965c8b790a760..e76317391ac34b 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java @@ -31,7 +31,7 @@ public class SiblingsUtilsTest { public void testGetSiblingUrns() { UrnArray siblingUrns = new UrnArray(UrnUtils.getUrn(TEST_DATASET_URN2), UrnUtils.getUrn(TEST_DATASET_URN3)); - EntityService mockService = mock(EntityService.class); + EntityService mockService = mock(EntityService.class); Mockito.when( mockService.getLatestAspect( any(), eq(UrnUtils.getUrn(TEST_DATASET_URN1)), eq(SIBLINGS_ASPECT_NAME))) @@ -45,7 +45,7 @@ public void testGetSiblingUrns() { @Test public void testGetSiblingUrnsWithoutSiblings() { - EntityService mockService = mock(EntityService.class); + EntityService mockService = mock(EntityService.class); Mockito.when( mockService.getLatestAspect( any(), eq(UrnUtils.getUrn(TEST_DATASET_URN1)), eq(SIBLINGS_ASPECT_NAME))) @@ -59,7 +59,7 @@ public void testGetSiblingUrnsWithoutSiblings() { @Test public void testGetSiblingUrnsWithSiblingsAspect() { - EntityService mockService = mock(EntityService.class); + EntityService mockService = mock(EntityService.class); Mockito.when( mockService.getLatestAspect( any(), eq(UrnUtils.getUrn(TEST_DATASET_URN1)), eq(SIBLINGS_ASPECT_NAME))) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolverTest.java index ce339fa2c75c7c..313c15c95c952f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolverTest.java @@ -23,7 +23,7 @@ public class UpdateUserSettingResolverTest { @Test public void testWriteCorpUserSettings() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_USER_URN)), eq(true))) .thenReturn(true); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolverTest.java index ad5f39d4b3c6d2..c4778cbbd40535 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolverTest.java @@ -57,8 +57,7 @@ public void testGetSuccess() throws Exception { Mockito.when(mockEnv.getContext()).thenReturn(mockContext); resolver.get(mockEnv).get(); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(expectedProposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, expectedProposal); } @Test diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java index 40a2e45724381c..b239e0300ffcc5 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java @@ -38,7 +38,7 @@ public class AddOwnersResolverTest { @Test public void testGetSuccessNoExistingOwners() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -101,7 +101,7 @@ public void testGetSuccessNoExistingOwners() throws Exception { @Test public void testGetSuccessExistingOwnerNewType() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); com.linkedin.common.Ownership oldOwnership = new Ownership() @@ -167,7 +167,7 @@ public void testGetSuccessExistingOwnerNewType() throws Exception { @Test public void testGetSuccessDeprecatedTypeToOwnershipType() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); com.linkedin.common.Ownership oldOwnership = new Ownership() @@ -230,7 +230,7 @@ public void testGetSuccessDeprecatedTypeToOwnershipType() throws Exception { @Test public void testGetSuccessMultipleOwnerTypes() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); com.linkedin.common.Ownership oldOwnership = new Ownership() @@ -328,7 +328,7 @@ public void testGetSuccessMultipleOwnerTypes() throws Exception { @Test public void testGetFailureOwnerDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -366,7 +366,7 @@ public void testGetFailureOwnerDoesNotExist() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -404,7 +404,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); AddOwnersResolver resolver = new AddOwnersResolver(mockService); @@ -429,7 +429,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java index 2d7b67685cc697..8275f9f83ef83f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java @@ -38,7 +38,7 @@ public class BatchAddOwnersResolverTest { @Test public void testGetSuccessNoExistingOwners() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -126,7 +126,7 @@ public void testGetSuccessExistingOwners() throws Exception { new Owner() .setOwner(Urn.createFromString(TEST_OWNER_URN_1)) .setType(OwnershipType.TECHNICAL_OWNER)))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -217,7 +217,7 @@ public void testGetSuccessExistingOwners() throws Exception { @Test public void testGetFailureOwnerDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -267,7 +267,7 @@ public void testGetFailureOwnerDoesNotExist() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -326,7 +326,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); @@ -363,7 +363,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchRemoveOwnersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchRemoveOwnersResolverTest.java index 8fcedfa605e1bc..9ea9ac693b98ed 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchRemoveOwnersResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchRemoveOwnersResolverTest.java @@ -35,7 +35,7 @@ public class BatchRemoveOwnersResolverTest { @Test public void testGetSuccessNoExistingOwners() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -83,7 +83,7 @@ public void testGetSuccessNoExistingOwners() throws Exception { @Test public void testGetSuccessExistingOwners() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); final Ownership oldOwners1 = new Ownership() @@ -150,7 +150,7 @@ public void testGetSuccessExistingOwners() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -195,7 +195,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchRemoveOwnersResolver resolver = new BatchRemoveOwnersResolver(mockService); @@ -218,7 +218,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java index 4384d9e2650e4f..ee1d59cdf87c75 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java @@ -35,7 +35,7 @@ public class AddTagsResolverTest { @Test public void testGetSuccessNoExistingTags() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -94,7 +94,7 @@ public void testGetSuccessExistingTags() throws Exception { ImmutableList.of( new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_1_URN))))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -146,7 +146,7 @@ public void testGetSuccessExistingTags() throws Exception { @Test public void testGetFailureTagDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -177,7 +177,7 @@ public void testGetFailureTagDoesNotExist() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -208,7 +208,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); AddTagsResolver resolver = new AddTagsResolver(mockService); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java index 0d22cd6e5eb2b1..5f6db4cb1e5a56 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java @@ -39,7 +39,7 @@ public class BatchAddTagsResolverTest { @Test public void testGetSuccessNoExistingTags() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -115,7 +115,7 @@ public void testGetSuccessExistingTags() throws Exception { ImmutableList.of( new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_1_URN))))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -184,7 +184,7 @@ public void testGetSuccessExistingTags() throws Exception { @Test public void testGetFailureTagDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -218,7 +218,7 @@ public void testGetFailureTagDoesNotExist() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -263,7 +263,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchAddTagsResolver resolver = new BatchAddTagsResolver(mockService); @@ -286,7 +286,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java index e6c33ea7341788..9f34c0da82744a 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java @@ -42,7 +42,7 @@ public class BatchRemoveTagsResolverTest { @Test public void testGetSuccessNoExistingTags() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -104,7 +104,7 @@ public void testGetSuccessNoExistingTags() throws Exception { @Test public void testGetSuccessExistingTags() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); final GlobalTags oldTags1 = new GlobalTags() @@ -177,7 +177,7 @@ public void testGetSuccessExistingTags() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -222,7 +222,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchRemoveTagsResolver resolver = new BatchRemoveTagsResolver(mockService); @@ -245,7 +245,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java index cd9ac9f0b610a8..6c8984addb2651 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java @@ -27,7 +27,7 @@ public class CreateTagResolverTest { @Test public void testGetSuccess() throws Exception { // Create resolver - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.when( mockClient.ingestProposal( @@ -53,14 +53,13 @@ public void testGetSuccess() throws Exception { key, TAG_ENTITY_NAME, TAG_PROPERTIES_ASPECT_NAME, props); // Not ideal to match against "any", but we don't know the auto-generated execution request id - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); } @Test public void testGetUnauthorized() throws Exception { // Create resolver - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); CreateTagResolver resolver = new CreateTagResolver(mockClient, mockService); @@ -77,7 +76,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { // Create resolver - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.doThrow(RuntimeException.class) .when(mockClient) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolverTest.java index 624e300b19f461..92f7a2688b43eb 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolverTest.java @@ -38,7 +38,7 @@ public class SetTagColorResolverTest { public void testGetSuccessExistingProperties() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); // Test setting the domain final TagProperties oldTagProperties = new TagProperties().setName("Test Tag"); @@ -69,8 +69,7 @@ public void testGetSuccessExistingProperties() throws Exception { MutationUtils.buildMetadataChangeProposalWithUrn( UrnUtils.getUrn(TEST_ENTITY_URN), TAG_PROPERTIES_ASPECT_NAME, newTagProperties); - Mockito.verify(mockClient, Mockito.times(1)) - .ingestProposal(any(), Mockito.eq(proposal), Mockito.eq(false)); + verifyIngestProposal(mockClient, 1, proposal); Mockito.verify(mockService, Mockito.times(1)) .exists(any(), Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true)); @@ -80,7 +79,7 @@ public void testGetSuccessExistingProperties() throws Exception { public void testGetFailureNoExistingProperties() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); // Test setting the domain Mockito.when( @@ -135,7 +134,7 @@ public void testGetFailureTagDoesNotExist() throws Exception { ImmutableMap.of( Constants.TAG_PROPERTIES_ASPECT_NAME, oldTagPropertiesAspect))))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when(mockService.exists(any(), eq(Urn.createFromString(TEST_ENTITY_URN)), eq(true))) .thenReturn(false); @@ -155,7 +154,7 @@ public void testGetFailureTagDoesNotExist() throws Exception { public void testGetUnauthorized() throws Exception { // Create resolver EntityClient mockClient = Mockito.mock(EntityClient.class); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); SetTagColorResolver resolver = new SetTagColorResolver(mockClient, mockService); // Execute resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/AddTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/AddTermsResolverTest.java index 9eeb525c3657e9..8f8a071ce89329 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/AddTermsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/AddTermsResolverTest.java @@ -32,7 +32,7 @@ public class AddTermsResolverTest { @Test public void testGetSuccessNoExistingTerms() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -82,7 +82,7 @@ public void testGetSuccessExistingTerms() throws Exception { new GlossaryTermAssociation() .setUrn(GlossaryTermUrn.createFromString(TEST_TERM_1_URN))))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -124,7 +124,7 @@ public void testGetSuccessExistingTerms() throws Exception { @Test public void testGetFailureTermDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -156,7 +156,7 @@ public void testGetFailureTermDoesNotExist() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -188,7 +188,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); AddTermsResolver resolver = new AddTermsResolver(mockService); @@ -207,7 +207,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchAddTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchAddTermsResolverTest.java index 185bb5cc97953a..ced9e371814f7e 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchAddTermsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchAddTermsResolverTest.java @@ -35,7 +35,7 @@ public class BatchAddTermsResolverTest { @Test public void testGetSuccessNoExistingTerms() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -99,7 +99,7 @@ public void testGetSuccessExistingTerms() throws Exception { new GlossaryTermAssociation() .setUrn(GlossaryTermUrn.createFromString(TEST_GLOSSARY_TERM_1_URN))))); - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -155,7 +155,7 @@ public void testGetSuccessExistingTerms() throws Exception { @Test public void testGetFailureTagDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -189,7 +189,7 @@ public void testGetFailureTagDoesNotExist() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -234,7 +234,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchAddTermsResolver resolver = new BatchAddTermsResolver(mockService); @@ -256,7 +256,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchRemoveTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchRemoveTermsResolverTest.java index 9e269bed436d98..254a301159ac25 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchRemoveTermsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchRemoveTermsResolverTest.java @@ -35,7 +35,7 @@ public class BatchRemoveTermsResolverTest { @Test public void testGetSuccessNoExistingTerms() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -82,7 +82,7 @@ public void testGetSuccessNoExistingTerms() throws Exception { @Test public void testGetSuccessExistingTerms() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); final GlossaryTerms oldTerms1 = new GlossaryTerms() @@ -148,7 +148,7 @@ public void testGetSuccessExistingTerms() throws Exception { @Test public void testGetFailureResourceDoesNotExist() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.when( mockService.getAspect( @@ -192,7 +192,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { @Test public void testGetUnauthorized() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); BatchRemoveTermsResolver resolver = new BatchRemoveTermsResolver(mockService); @@ -214,7 +214,7 @@ public void testGetUnauthorized() throws Exception { @Test public void testGetEntityClientException() throws Exception { - EntityService mockService = getMockEntityService(); + EntityService mockService = getMockEntityService(); Mockito.doThrow(RuntimeException.class) .when(mockService) diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/browsepaths/BackfillBrowsePathsV2Step.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/browsepaths/BackfillBrowsePathsV2Step.java index 09b27fbcfbdf29..a1d559d05ad2fb 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/browsepaths/BackfillBrowsePathsV2Step.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/browsepaths/BackfillBrowsePathsV2Step.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.upgrade.system.browsepaths; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -29,7 +30,6 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; -import com.linkedin.mxe.SystemMetadata; import io.datahubproject.metadata.context.OperationContext; import java.util.Set; import java.util.function.Function; @@ -208,8 +208,7 @@ private void ingestBrowsePathsV2( proposal.setEntityType(urn.getEntityType()); proposal.setAspectName(Constants.BROWSE_PATHS_V2_ASPECT_NAME); proposal.setChangeType(ChangeType.UPSERT); - proposal.setSystemMetadata( - new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(System.currentTimeMillis())); + proposal.setSystemMetadata(createDefaultSystemMetadata()); proposal.setAspect(GenericRecordUtils.serializeAspect(browsePathsV2)); entityService.ingestProposal(opContext, proposal, auditStamp, true); } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java index 93bf8cc5b9b60e..c65a45aefc357f 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.upgrade.system.policyfields; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; import com.google.common.collect.ImmutableList; import com.linkedin.common.AuditStamp; @@ -25,7 +26,6 @@ import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchService; -import com.linkedin.mxe.SystemMetadata; import com.linkedin.policy.DataHubPolicyInfo; import io.datahubproject.metadata.context.OperationContext; import java.net.URISyntaxException; @@ -234,9 +234,7 @@ private Optional> ingestPolicyFields( null, infoAspect, null, - new SystemMetadata() - .setRunId(DEFAULT_RUN_ID) - .setLastObserved(System.currentTimeMillis()), + createDefaultSystemMetadata(), auditStamp, ChangeType.RESTATE) .getFirst()); diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx index 5be31528fe780f..2fdd7c8ed68004 100644 --- a/datahub-web-react/src/App.tsx +++ b/datahub-web-react/src/App.tsx @@ -1,6 +1,5 @@ import React from 'react'; import Cookies from 'js-cookie'; -import { message } from 'antd'; import { BrowserRouter as Router } from 'react-router-dom'; import { ApolloClient, ApolloProvider, createHttpLink, InMemoryCache, ServerError } from '@apollo/client'; import { onError } from '@apollo/client/link/error'; @@ -21,7 +20,7 @@ import { useCustomTheme } from './customThemeContext'; const httpLink = createHttpLink({ uri: '/api/v2/graphql' }); const errorLink = onError((error) => { - const { networkError, graphQLErrors } = error; + const { networkError } = error; if (networkError) { const serverError = networkError as ServerError; if (serverError.statusCode === ErrorCodes.Unauthorized) { @@ -31,13 +30,14 @@ const errorLink = onError((error) => { window.location.replace(`${PageRoutes.AUTHENTICATE}?redirect_uri=${encodeURIComponent(currentPath)}`); } } - if (graphQLErrors && graphQLErrors.length) { - const firstError = graphQLErrors[0]; - const { extensions } = firstError; - const errorCode = extensions && (extensions.code as number); - // Fallback in case the calling component does not handle. - message.error(`${firstError.message} (code ${errorCode})`, 3); - } + // Disabled behavior for now -> Components are expected to handle their errors. + // if (graphQLErrors && graphQLErrors.length) { + // const firstError = graphQLErrors[0]; + // const { extensions } = firstError; + // const errorCode = extensions && (extensions.code as number); + // // Fallback in case the calling component does not handle. + // message.error(`${firstError.message} (code ${errorCode})`, 3); // TODO: Decide if we want this back. + // } }); const client = new ApolloClient({ diff --git a/datahub-web-react/src/graphql/view.graphql b/datahub-web-react/src/graphql/view.graphql index 9d38a09b8280bc..f7b82a1e46f046 100644 --- a/datahub-web-react/src/graphql/view.graphql +++ b/datahub-web-react/src/graphql/view.graphql @@ -39,6 +39,12 @@ query listGlobalViews($start: Int!, $count: Int!, $query: String) { } } +query getView($urn: String!) { + view(urn: $urn) { + ...view + } +} + mutation createView($input: CreateViewInput!) { createView(input: $input) { ...view diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile index a828f1d8c27ad5..2a9354cbf6a04f 100644 --- a/docker/datahub-frontend/Dockerfile +++ b/docker/datahub-frontend/Dockerfile @@ -25,24 +25,25 @@ RUN apk --no-cache --update-cache --available upgrade \ ENV LD_LIBRARY_PATH="/lib:/lib64" -FROM base as prod-install +FROM base as unpack COPY ./datahub-frontend.zip / -RUN unzip datahub-frontend.zip -d /datahub-frontend \ - && mv /datahub-frontend/main/* /datahub-frontend \ - && rmdir /datahub-frontend/main \ - && rm datahub-frontend.zip +RUN unzip datahub-frontend.zip -d /tmp/out \ + && mv /tmp/out/main /datahub-frontend COPY ./docker/monitoring/client-prometheus-config.yaml /datahub-frontend/ RUN chown -R datahub:datahub /datahub-frontend && chmod 755 /datahub-frontend +FROM base as prod-install + +COPY --from=unpack /datahub-frontend/ /datahub-frontend/ + FROM base as dev-install # Dummy stage for development. Assumes code is built on your machine and mounted to this image. # See this excellent thread https://github.com/docker/cli/issues/1134 VOLUME [ "/datahub-frontend" ] FROM ${APP_ENV}-install as final -COPY ./docker/datahub-frontend/start.sh / -RUN chown datahub:datahub /start.sh && chmod 755 /start.sh +COPY --chown=datahub:datahub --chmod=755 ./docker/datahub-frontend/start.sh / USER datahub ARG SERVER_PORT=9002 diff --git a/docker/datahub-gms/env/docker-without-neo4j.env b/docker/datahub-gms/env/docker-without-neo4j.env index cc0dd6b4278b56..37b7ba1797af5b 100644 --- a/docker/datahub-gms/env/docker-without-neo4j.env +++ b/docker/datahub-gms/env/docker-without-neo4j.env @@ -23,6 +23,8 @@ PE_CONSUMER_ENABLED=true UI_INGESTION_ENABLED=true ENTITY_SERVICE_ENABLE_RETENTION=true +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to disable persistence of client-side analytics events # DATAHUB_ANALYTICS_ENABLED=false diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env index 59fc4bdde02ff4..0ecaa32c4cb123 100644 --- a/docker/datahub-gms/env/docker.env +++ b/docker/datahub-gms/env/docker.env @@ -27,6 +27,8 @@ MCE_CONSUMER_ENABLED=true PE_CONSUMER_ENABLED=true UI_INGESTION_ENABLED=true +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to enable Metadata Service Authentication METADATA_SERVICE_AUTH_ENABLED=false diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index 383478b675640f..8a238c32704bb6 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -85,18 +85,18 @@ RUN apt-get update && apt-get install -y -qq \ RUN if [ $(arch) = "x86_64" ]; then \ mkdir /opt/oracle && \ cd /opt/oracle && \ - wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/216000/instantclient-basic-linux.x64-21.6.0.0.0dbru.zip && \ - unzip instantclient-basic-linux.x64-21.6.0.0.0dbru.zip && \ - rm instantclient-basic-linux.x64-21.6.0.0.0dbru.zip && \ - sh -c "echo /opt/oracle/instantclient_21_6 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ + wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/2115000/instantclient-basic-linux.x64-21.15.0.0.0dbru.zip && \ + unzip instantclient-basic-linux.x64-21.15.0.0.0dbru.zip && \ + rm instantclient-basic-linux.x64-21.15.0.0.0dbru.zip && \ + sh -c "echo /opt/oracle/instantclient_21_15 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ ldconfig; \ else \ mkdir /opt/oracle && \ cd /opt/oracle && \ - wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/191000/instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip && \ - unzip instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip && \ - rm instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip && \ - sh -c "echo /opt/oracle/instantclient_19_10 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ + wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/1923000/instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip && \ + unzip instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip && \ + rm instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip && \ + sh -c "echo /opt/oracle/instantclient_19_23 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \ ldconfig; \ fi; diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt index 2f2b64723f0edd..fa07b4184a6bc0 100644 --- a/docker/datahub-ingestion-base/base-requirements.txt +++ b/docker/datahub-ingestion-base/base-requirements.txt @@ -1,19 +1,20 @@ # Generated requirements file. Run ./regenerate-base-requirements.sh to regenerate. -acryl-datahub-classify==0.0.10 +acryl-datahub-classify==0.0.11 acryl-PyHive==0.6.16 -acryl-sqlglot==24.0.1.dev7 +acryl-sqlglot==25.3.1.dev3 aenum==3.1.15 -aiohttp==3.9.5 +aiohappyeyeballs==2.3.2 +aiohttp==3.10.0 aiosignal==1.3.1 -alembic==1.13.1 +alembic==1.13.2 altair==4.2.0 anyio==4.4.0 -apache-airflow==2.9.2 +apache-airflow==2.9.3 apache-airflow-providers-common-io==1.3.2 -apache-airflow-providers-common-sql==1.14.0 -apache-airflow-providers-fab==1.1.1 -apache-airflow-providers-ftp==3.9.1 -apache-airflow-providers-http==4.11.1 +apache-airflow-providers-common-sql==1.14.2 +apache-airflow-providers-fab==1.2.2 +apache-airflow-providers-ftp==3.10.0 +apache-airflow-providers-http==4.12.0 apache-airflow-providers-imap==3.6.1 apache-airflow-providers-smtp==1.7.1 apache-airflow-providers-sqlite==3.8.1 @@ -26,25 +27,30 @@ asgiref==3.8.1 asn1crypto==1.5.1 asttokens==2.4.1 async-timeout==4.0.3 -asynch==0.2.3 +asynch==0.2.4 attrs==23.2.0 avro==1.11.3 avro-gen3==0.7.13 +azure-common==1.1.28 +azure-core==1.29.4 +azure-identity==1.14.1 +azure-storage-blob==12.21.0 +azure-storage-file-datalake==12.16.0 Babel==2.15.0 backoff==2.2.1 beautifulsoup4==4.12.3 bleach==6.1.0 blinker==1.8.2 blis==0.7.11 -boto3==1.34.129 -botocore==1.34.129 +boto3==1.34.151 +botocore==1.34.151 bracex==2.4 cached-property==1.5.2 cachelib==0.9.0 -cachetools==5.3.3 +cachetools==5.4.0 catalogue==2.0.10 cattrs==23.2.3 -certifi==2024.6.2 +certifi==2024.7.4 cffi==1.16.0 chardet==5.2.0 charset-normalizer==3.3.2 @@ -55,25 +61,26 @@ click-spinner==0.1.10 clickclick==20.10.2 clickhouse-driver==0.2.8 clickhouse-sqlalchemy==0.2.4 +cloudpathlib==0.18.1 cloudpickle==3.0.0 colorama==0.4.6 colorlog==4.8.0 comm==0.2.2 confection==0.1.5 ConfigUpdater==3.2 -confluent-kafka==2.4.0 -connexion==2.14.1 +confluent-kafka==2.5.0 +connexion==2.14.2 cron-descriptor==1.4.3 -croniter==2.0.5 +croniter==3.0.3 cryptography==42.0.8 cx_Oracle==8.3.0 cymem==2.0.8 databricks-dbapi==0.6.0 -databricks-sdk==0.28.0 +databricks-sdk==0.29.0 databricks-sql-connector==2.9.6 dataflows-tabulator==1.54.3 db-dtypes==1.2.0 -debugpy==1.8.1 +debugpy==1.8.2 decorator==5.1.1 defusedxml==0.7.1 deltalake==0.17.4 @@ -84,42 +91,42 @@ docker==7.1.0 docutils==0.21.2 ecdsa==0.19.0 elasticsearch==7.13.4 -email_validator==2.1.2 +email_validator==2.2.0 entrypoints==0.4 et-xmlfile==1.1.0 -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 executing==2.0.1 expandvars==0.12.0 -fastavro==1.9.4 +fastavro==1.9.5 fastjsonschema==2.20.0 -filelock==3.15.1 +filelock==3.15.4 Flask==2.2.5 flatdict==4.0.1 frozenlist==1.4.1 fsspec==2023.12.2 future==1.0.0 -GeoAlchemy2==0.15.1 +GeoAlchemy2==0.15.2 gitdb==4.0.11 GitPython==3.1.43 -google-api-core==2.19.0 -google-auth==2.30.0 -google-cloud-appengine-logging==1.4.3 +google-api-core==2.19.1 +google-auth==2.32.0 +google-cloud-appengine-logging==1.4.5 google-cloud-audit-log==0.2.5 -google-cloud-bigquery==3.24.0 +google-cloud-bigquery==3.25.0 google-cloud-core==2.4.1 -google-cloud-datacatalog==3.19.0 +google-cloud-datacatalog==3.20.0 google-cloud-datacatalog-lineage==0.2.2 google-cloud-logging==3.5.0 google-crc32c==1.5.0 -google-re2==1.1.20240601 +google-re2==1.1.20240702 google-resumable-media==2.7.1 -googleapis-common-protos==1.63.1 +googleapis-common-protos==1.63.2 gql==3.5.0 graphql-core==3.2.3 great-expectations==0.15.50 greenlet==3.0.3 -grpc-google-iam-v1==0.13.0 -grpcio==1.64.1 +grpc-google-iam-v1==0.13.1 +grpcio==1.65.2 grpcio-status==1.62.2 grpcio-tools==1.62.2 gssapi==1.8.3 @@ -130,7 +137,7 @@ httpx==0.27.0 humanfriendly==10.0 idna==3.7 ijson==3.3.0 -importlib_metadata==7.1.0 +importlib_metadata==7.2.1 importlib_resources==6.4.0 inflection==0.5.1 ipaddress==1.0.23 @@ -148,7 +155,7 @@ jsonlines==4.0.0 jsonpatch==1.33 jsonpointer==3.0.0 jsonref==1.1.0 -jsonschema==4.22.0 +jsonschema==4.23.0 jsonschema-specifications==2023.12.1 jupyter-server==1.16.0 jupyter_client==7.4.9 @@ -159,16 +166,16 @@ langcodes==3.4.0 language_data==1.2.0 lark==1.1.4 lazy-object-proxy==1.10.0 -leb128==1.0.7 -limits==3.12.0 +leb128==1.0.8 +limits==3.13.0 linear-tsv==1.1.0 linkify-it-py==2.0.3 -lkml==1.3.4 +lkml==1.3.5 lockfile==0.12.2 looker-sdk==23.0.0 lxml==5.2.2 lz4==4.3.3 -makefun==1.15.2 +makefun==1.15.4 Mako==1.3.5 marisa-trie==1.2.0 markdown-it-py==3.0.0 @@ -182,11 +189,12 @@ mdurl==0.1.2 methodtools==0.4.7 mistune==3.0.2 mixpanel==4.10.1 -mlflow-skinny==2.14.0 +mlflow-skinny==2.15.0 mmhash3==3.0.1 more-itertools==10.3.0 moto==4.2.14 msal==1.22.0 +msal-extensions==1.1.0 multidict==6.0.5 murmurhash==1.0.10 mypy-extensions==1.0.0 @@ -201,47 +209,46 @@ notebook_shim==0.2.4 numpy==1.26.4 oauthlib==3.2.2 okta==1.7.0 -openlineage-airflow==1.12.0 -openlineage-integration-common==1.12.0 -openlineage-python==1.12.0 -openlineage_sql==1.12.0 -openpyxl==3.1.4 -opentelemetry-api==1.25.0 -opentelemetry-exporter-otlp==1.25.0 -opentelemetry-exporter-otlp-proto-common==1.25.0 -opentelemetry-exporter-otlp-proto-grpc==1.25.0 -opentelemetry-exporter-otlp-proto-http==1.25.0 -opentelemetry-proto==1.25.0 -opentelemetry-sdk==1.25.0 -opentelemetry-semantic-conventions==0.46b0 +openlineage-airflow==1.18.0 +openlineage-integration-common==1.18.0 +openlineage-python==1.18.0 +openlineage_sql==1.18.0 +openpyxl==3.1.5 +opentelemetry-api==1.26.0 +opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-proto==1.26.0 +opentelemetry-sdk==1.26.0 +opentelemetry-semantic-conventions==0.47b0 ordered-set==4.1.0 packaging==24.1 pandas==2.1.4 pandocfilters==1.5.1 parse==1.20.2 parso==0.8.4 -pathlib_abc==0.1.1 pathspec==0.12.1 -pathy==0.11.0 pendulum==3.0.0 pexpect==4.9.0 phonenumbers==8.13.0 platformdirs==4.2.2 pluggy==1.5.0 +portalocker==2.10.1 preshed==3.0.9 prison==0.2.1 progressbar2==4.4.2 prometheus_client==0.20.0 prompt_toolkit==3.0.47 -proto-plus==1.23.0 -protobuf==4.25.3 +proto-plus==1.24.0 +protobuf==4.25.4 psutil==6.0.0 psycopg2-binary==2.9.9 ptyprocess==0.7.0 -pure-eval==0.2.2 pure-sasl==0.6.2 +pure_eval==0.2.3 py-partiql-parser==0.5.0 -pyarrow==16.1.0 +pyarrow==17.0.0 pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 @@ -249,16 +256,16 @@ pyathena==2.25.2 pycountry==24.6.1 pycparser==2.22 pycryptodome==3.20.0 -pydantic==1.10.16 -pydash==8.0.1 +pydantic==1.10.17 +pydash==8.0.3 pydruid==0.6.9 Pygments==2.18.0 pyiceberg==0.4.0 -pymongo==4.7.3 +pymongo==4.8.0 PyMySQL==1.1.1 -pyOpenSSL==24.1.0 +pyOpenSSL==24.2.1 pyparsing==3.0.9 -pyspnego==0.11.0 +pyspnego==0.11.1 python-daemon==3.0.1 python-dateutil==2.9.0.post0 python-jose==3.3.0 @@ -273,9 +280,9 @@ pytz==2024.1 PyYAML==6.0.1 pyzmq==26.0.3 redash-toolbelt==0.1.9 -redshift-connector==2.1.1 +redshift-connector==2.1.2 referencing==0.35.1 -regex==2024.5.15 +regex==2024.7.24 requests==2.32.3 requests-file==2.1.0 requests-gssapi==1.3.0 @@ -286,31 +293,32 @@ rfc3339-validator==0.1.4 rfc3986==2.0.0 rich==13.7.1 rich-argparse==1.5.2 -rpds-py==0.18.1 +rpds-py==0.19.1 rsa==4.9 rstr==3.2.2 ruamel.yaml==0.17.17 -s3transfer==0.10.1 +s3transfer==0.10.2 schwifty==2024.6.1 -scipy==1.13.1 +scipy==1.14.0 scramp==1.4.5 Send2Trash==1.8.3 -sentry-sdk==2.5.1 +sentry-sdk==2.12.0 setproctitle==1.3.3 +shellingham==1.5.4 simple-salesforce==1.12.6 six==1.16.0 slack-sdk==3.18.1 -smart-open==6.4.0 +smart-open==7.0.4 smmap==5.0.1 sniffio==1.3.1 -snowflake-connector-python==3.11.0 -snowflake-sqlalchemy==1.5.3 +snowflake-connector-python==3.12.0 +snowflake-sqlalchemy==1.6.1 sortedcontainers==2.4.0 soupsieve==2.5 -spacy==3.5.0 +spacy==3.7.5 spacy-legacy==3.0.12 spacy-loggers==1.0.5 -sql-metadata==2.2.2 +sql_metadata==2.12.0 SQLAlchemy==1.4.44 sqlalchemy-bigquery==1.11.0 sqlalchemy-cockroachdb==1.4.4 @@ -318,7 +326,7 @@ SQLAlchemy-JSONField==1.0.2 sqlalchemy-pytds==0.3.5 sqlalchemy-redshift==0.8.14 SQLAlchemy-Utils==0.41.2 -sqlglotrs==0.2.5 +sqlglotrs==0.2.7 sqllineage==1.3.8 sqlparse==0.4.4 srsly==2.4.8 @@ -327,25 +335,25 @@ strictyaml==1.7.3 tableauserverclient==0.25 tableschema==1.20.11 tabulate==0.9.0 -tenacity==8.4.1 -teradatasql==20.0.0.12 +tenacity==9.0.0 +teradatasql==20.0.0.14 teradatasqlalchemy==20.0.0.1 termcolor==2.4.0 terminado==0.18.1 text-unidecode==1.3 -thinc==8.1.12 +thinc==8.2.5 thrift==0.16.0 thrift-sasl==0.4.3 -time-machine==2.14.1 +time-machine==2.14.2 tinycss2==1.3.0 toml==0.10.2 -tomlkit==0.12.5 +tomlkit==0.13.0 toolz==0.12.1 tornado==6.4.1 tqdm==4.66.4 traitlets==5.2.1.post0 -trino==0.328.0 -typer==0.7.0 +trino==0.329.0 +typer==0.12.3 typing-inspect==0.9.0 typing_extensions==4.12.2 tzdata==2024.1 @@ -355,15 +363,16 @@ ujson==5.10.0 unicodecsv==0.14.1 universal_pathlib==0.2.2 urllib3==1.26.19 -vertica-python==1.3.8 +vertica-python==1.4.0 vertica-sqlalchemy-dialect==0.0.8.2 vininfo==1.8.0 wasabi==1.1.3 wcmatch==8.5.2 wcwidth==0.2.13 +weasel==0.4.1 webencodings==0.5.1 websocket-client==1.8.0 -Werkzeug==2.3.8 +Werkzeug==2.2.3 widgetsnbextension==4.0.11 wirerope==0.4.7 wrapt==1.16.0 diff --git a/docker/datahub-ingestion-base/build.gradle b/docker/datahub-ingestion-base/build.gradle index faa0589cfbfbbf..5652fedcd93b3b 100644 --- a/docker/datahub-ingestion-base/build.gradle +++ b/docker/datahub-ingestion-base/build.gradle @@ -12,7 +12,7 @@ ext { docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}" - revision = 3 // increment to trigger rebuild + revision = 4 // increment to trigger rebuild } docker { diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile index 068911695811f5..b8eda548491224 100644 --- a/docker/datahub-ingestion/Dockerfile +++ b/docker/datahub-ingestion/Dockerfile @@ -1,7 +1,7 @@ # Defining environment ARG APP_ENV=full ARG BASE_IMAGE=acryldata/datahub-ingestion-base -ARG DOCKER_VERSION=head +ARG DOCKER_VERSION=head-full ARG DEBIAN_REPO_URL=https://deb.debian.org/debian ARG PIP_MIRROR_URL=https://pypi.python.org/simple diff --git a/docker/datahub-ingestion/build.gradle b/docker/datahub-ingestion/build.gradle index b9ab546674a031..6757be7cd6f221 100644 --- a/docker/datahub-ingestion/build.gradle +++ b/docker/datahub-ingestion/build.gradle @@ -12,7 +12,7 @@ ext { docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}" - revision = 3 // increment to trigger rebuild + revision = 4 // increment to trigger rebuild } dependencies { diff --git a/docker/datahub-mae-consumer/env/docker-without-neo4j.env b/docker/datahub-mae-consumer/env/docker-without-neo4j.env index b6899f7e6d63b2..6a82f235b29711 100644 --- a/docker/datahub-mae-consumer/env/docker-without-neo4j.env +++ b/docker/datahub-mae-consumer/env/docker-without-neo4j.env @@ -13,6 +13,8 @@ ES_BULK_REFRESH_POLICY=WAIT_UNTIL GRAPH_SERVICE_IMPL=elasticsearch ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to disable persistence of client-side analytics events # DATAHUB_ANALYTICS_ENABLED=false diff --git a/docker/datahub-mae-consumer/env/docker.env b/docker/datahub-mae-consumer/env/docker.env index 5a6daa6eaeaed7..1f0ee4b05b3820 100644 --- a/docker/datahub-mae-consumer/env/docker.env +++ b/docker/datahub-mae-consumer/env/docker.env @@ -17,6 +17,8 @@ NEO4J_PASSWORD=datahub GRAPH_SERVICE_IMPL=neo4j ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to disable persistence of client-side analytics events # DATAHUB_ANALYTICS_ENABLED=false diff --git a/docker/datahub-mce-consumer/env/docker-without-neo4j.env b/docker/datahub-mce-consumer/env/docker-without-neo4j.env index e7be7d8ed4ddc5..b0edfc0a75b669 100644 --- a/docker/datahub-mce-consumer/env/docker-without-neo4j.env +++ b/docker/datahub-mce-consumer/env/docker-without-neo4j.env @@ -24,6 +24,8 @@ MAE_CONSUMER_ENABLED=false PE_CONSUMER_ENABLED=false UI_INGESTION_ENABLED=false +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment # METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env index 8618f3f5f7af7a..c0f85ef667546e 100644 --- a/docker/datahub-mce-consumer/env/docker.env +++ b/docker/datahub-mce-consumer/env/docker.env @@ -24,6 +24,8 @@ MAE_CONSUMER_ENABLED=false PE_CONSUMER_ENABLED=false UI_INGESTION_ENABLED=false +ELASTIC_ID_HASH_ALGO=MD5 + # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment # METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index d722b07b9a7af4..de766f76cb626e 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -144,7 +144,7 @@ services: - neo4jdata:/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -159,7 +159,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - 29092:29092 - 9092:9092 @@ -177,7 +177,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - 2181:2181 env_file: zookeeper/env/docker.env diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index eae36fb849fd5c..748a2cc9e04167 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -123,7 +123,7 @@ services: - esdata:/usr/share/elasticsearch/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -138,7 +138,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 env_file: broker/env/docker.env @@ -155,7 +155,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 env_file: zookeeper/env/docker.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 96f37496859a46..ae55861580becd 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -143,7 +143,7 @@ services: - neo4jdata:/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -158,7 +158,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 env_file: broker/env/docker.env @@ -175,7 +175,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 env_file: zookeeper/env/docker.env diff --git a/docker/kafka-setup/env_to_properties.py b/docker/kafka-setup/env_to_properties.py new file mode 100644 index 00000000000000..8d8b8c3cc7b59f --- /dev/null +++ b/docker/kafka-setup/env_to_properties.py @@ -0,0 +1,24 @@ +import os +import re +import sys + + +def env_to_properties(env_prefix: str, properties_file: str): + pattern = re.compile('(?<=[^_])_(?=[^_])') + props = {} + + for (env_name, val) in os.environ.items(): + if env_name.startswith(env_prefix): + raw_name = env_name[len(env_prefix):].lower() + prop_dot = '.'.join(pattern.split(raw_name)) + props[prop_dot] = val + + with open(properties_file, 'a') as f: + for k, v in props.items(): + f.writelines(f'{k}={v}\n') + + +if __name__ == '__main__': + env_prefix = sys.argv[1] + properties_file = sys.argv[2] + env_to_properties(env_prefix, properties_file) diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh index 439ffb4d4d8295..392cca94666419 100755 --- a/docker/kafka-setup/kafka-setup.sh +++ b/docker/kafka-setup/kafka-setup.sh @@ -10,46 +10,8 @@ fi . kafka-config.sh echo "bootstrap.servers=$KAFKA_BOOTSTRAP_SERVER" > $CONNECTION_PROPERTIES_PATH -echo "security.protocol=$KAFKA_PROPERTIES_SECURITY_PROTOCOL" >> $CONNECTION_PROPERTIES_PATH -## Add support for SASL_PLAINTEXT -if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SASL_PLAINTEXT" ]]; then - echo "sasl.mechanism=$KAFKA_PROPERTIES_SASL_MECHANISM" >> $CONNECTION_PROPERTIES_PATH - echo "sasl.jaas.config=$KAFKA_PROPERTIES_SASL_JAAS_CONFIG" >> $CONNECTION_PROPERTIES_PATH - echo "sasl.kerberos.service.name=$KAFKA_PROPERTIES_SASL_KERBEROS_SERVICE_NAME" >> $CONNECTION_PROPERTIES_PATH -fi - -## Add support for SASL_SSL -if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SASL_SSL" ]]; then - echo "sasl.jaas.config=$KAFKA_PROPERTIES_SASL_JAAS_CONFIG" >> $CONNECTION_PROPERTIES_PATH - echo "sasl.mechanism=$KAFKA_PROPERTIES_SASL_MECHANISM" >> $CONNECTION_PROPERTIES_PATH -fi - -if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SSL" ]]; then - if [[ -n $KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION ]]; then - echo "ssl.keystore.location=$KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION" >> $CONNECTION_PROPERTIES_PATH - echo "ssl.keystore.password=$KAFKA_PROPERTIES_SSL_KEYSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH - echo "ssl.key.password=$KAFKA_PROPERTIES_SSL_KEY_PASSWORD" >> $CONNECTION_PROPERTIES_PATH - if [[ -n $KAFKA_PROPERTIES_SSL_KEYSTORE_TYPE ]]; then - echo "ssl.keystore.type=$KAFKA_PROPERTIES_SSL_KEYSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH - fi - fi - if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION ]]; then - echo "ssl.truststore.location=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION" >> $CONNECTION_PROPERTIES_PATH - if [[ $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE != "PEM" ]]; then - echo "ssl.truststore.password=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH - fi - if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE ]]; then - echo "ssl.truststore.type=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH - fi - fi - echo "ssl.endpoint.identification.algorithm=$KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM" >> $CONNECTION_PROPERTIES_PATH -fi - -# Add support for SASL_CLIENT_CALLBACK_HANDLER_CLASS -if [[ -n "$KAFKA_PROPERTIES_SASL_CLIENT_CALLBACK_HANDLER_CLASS" ]]; then - echo "sasl.client.callback.handler.class=$KAFKA_PROPERTIES_SASL_CLIENT_CALLBACK_HANDLER_CLASS" >> $CONNECTION_PROPERTIES_PATH -fi +python env_to_properties.py KAFKA_PROPERTIES_ $CONNECTION_PROPERTIES_PATH # cub kafka-ready -c $CONNECTION_PROPERTIES_PATH -b $KAFKA_BOOTSTRAP_SERVER 1 180 . kafka-ready.sh diff --git a/docker/profiles/docker-compose.prerequisites.yml b/docker/profiles/docker-compose.prerequisites.yml index 08ebc8b65d8c9c..7cd9c9039539cc 100644 --- a/docker/profiles/docker-compose.prerequisites.yml +++ b/docker/profiles/docker-compose.prerequisites.yml @@ -210,7 +210,7 @@ services: - neo4jdata:/data kafka-broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} command: - /bin/bash - -c diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 834d55096468f6..a0f60d23710a07 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -86,6 +86,7 @@ services: - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 47fb50f78e4f0c..11e33a9950ba9b 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -86,6 +86,7 @@ services: - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 3fa13a9e56b421..2efa8959834183 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -86,6 +86,7 @@ services: - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml index a4211acedcf102..4f47a3da24eb1b 100644 --- a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml @@ -19,6 +19,7 @@ services: - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - GRAPH_SERVICE_IMPL=elasticsearch - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml + - ELASTIC_ID_HASH_ALGO=MD5 hostname: datahub-mae-consumer image: ${DATAHUB_MAE_CONSUMER_IMAGE:-acryldata/datahub-mae-consumer}:${DATAHUB_VERSION:-head} ports: @@ -37,6 +38,7 @@ services: - EBEAN_DATASOURCE_USERNAME=datahub - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml index e7571e4baf8b4e..7dd7388b939884 100644 --- a/docker/quickstart/docker-compose.consumers.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers.quickstart.yml @@ -26,6 +26,7 @@ services: - NEO4J_PASSWORD=datahub - GRAPH_SERVICE_IMPL=neo4j - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml + - ELASTIC_ID_HASH_ALGO=MD5 hostname: datahub-mae-consumer image: ${DATAHUB_MAE_CONSUMER_IMAGE:-acryldata/datahub-mae-consumer}:${DATAHUB_VERSION:-head} ports: @@ -47,6 +48,7 @@ services: - EBEAN_DATASOURCE_USERNAME=datahub - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index c63b6d1d61b030..f42ed1f40c2467 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -86,6 +86,7 @@ services: - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_PORT=9200 + - ELASTIC_ID_HASH_ALGO=MD5 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_SERVICE_ENABLE_RETENTION=true - ES_BULK_REFRESH_POLICY=WAIT_UNTIL diff --git a/docs-website/adoptionStoriesIndexes.json b/docs-website/adoptionStoriesIndexes.json new file mode 100644 index 00000000000000..3fe666ccf1c134 --- /dev/null +++ b/docs-website/adoptionStoriesIndexes.json @@ -0,0 +1,330 @@ +{ + "companies": [ + { + "name": "Netflix", + "slug": "netflix", + "imageUrl": "/img/logos/companies/netflix.png", + "imageSize": "large", + "link": "https://blog.datahubproject.io/how-netflix-is-collaborating-with-datahub-to-enhance-its-extensibility-a34d33f45947", + "linkType": "blog", + "tagline": "How Netflix is collaborating with DataHub to enhance its extensibility", + "category": "B2B & B2C", + "description": "\"DataHub gave us the extensibility features we needed to define new entity types easily and augment existing ones. DataHub performed exceptionally well in managing our traffic load and data volume. It offers a great developer experience, a well-documented taxonomy, and — very importantly — solid community support.\"

— Ajoy Majumdar, Software Architect at Netflix

" + }, + { + "name": "Visa", + "slug": "visa", + "imageUrl": "/img/logos/companies/visa.png", + "imageSize": "large", + "link": "https://blog.datahubproject.io/how-visa-uses-datahub-to-scale-data-governance-cace052d61c5", + "linkType": "blog", + "tagline": "How Visa uses DataHub to scale data governance", + "category": "Financial & Fintech", + "description": "\"We found DataHub to provide excellent coverage for our needs. What we appreciate most about DataHub is its powerful API platform.\"

— Jean-Pierre Dijcks, Sr. Dir. Product Management at VISA

" + }, + { + "name": "Optum", + "slug": "optum", + "imageUrl": "/img/logos/companies/optum.jpg", + "imageSize": "medium", + "link": "https://opensource.optum.com/blog/2022/03/23/data-mesh-via-datahub", + "linkType": "blog", + "tagline": "Data Mesh via DataHub", + "category": "And More", + "description": "“DataHub’s event driven architecture provides us a mechanism to act on any metadata changes in real time. This allows us to perform various actions like provisioning access to a data product, notifying consumers on any schema changes that may affect their application or triggering data movement jobs to move data from source to sink platforms.”" + }, + { + "name": "Pinterest", + "slug": "pinterest", + "imageUrl": "/img/logos/companies/pinterest.png", + "imageSize": "small", + "link": "https://www.youtube.com/watch?v=YoxTg8tQSwg&feature=youtu.be", + "linkType": "blog", + "tagline": "DataHub Project at Pinterest", + "category": "B2B & B2C", + "description": "Pinterest adopted a DataHub project to enhance metadata management for its big data query platform, facilitating better data navigation and understanding." + }, + { + "name": "Airtel", + "slug": "airtel", + "imageUrl": "/img/logos/companies/airtel.png", + "imageSize": "large", + "link": "https://www.youtube.com/watch?v=yr24mM91BN4", + "linkType": "video", + "tagline": "A transformative journey to Airtel's data mesh architecture with DataHub", + "category": "B2B & B2C", + "description": "Airtel is a leading global telecommunication provider. DataHub is the bedrock of Data Mesh at Airtel by providing the requisite governance and metadata management functionality to ensure their Data Products are discoverable, addressable, trustworthy, self-describing, and secure.

Get a closer look at how the Airtel team has successfully integrated DataHub to take their data mesh implementation to the next level." + }, + { + "name": "Coursera", + "slug": "coursera", + "imageUrl": "/img/logos/companies/coursera.svg", + "imageSize": "small", + "link": "https://www.youtube.com/watch?v=bd5v4fn4d4s", + "linkType": "video", + "tagline": "Coursera's DataHub Journey", + "category": "B2B & B2C", + "description": "“DataHub aligns with our needs [for] data documentation, a unified search experience, lineage information, and additional metadata. We are also very impressed with the vibrant and supportive community.”" + }, + { + "name": "Zynga", + "slug": "zynga", + "imageUrl": "/img/logos/companies/zynga.png", + "imageSize": "default", + "link": "https://www.youtube.com/watch?v=VCU3-Hd_glI", + "linkType": "video", + "tagline": "Zynga's DataHub Implementation", + "category": "B2B & B2C", + "description": "“We looked around for data catalog tool, and DataHub was a clear winner.”

Zynga levels up data management using DataHub, highlighting its role in enhancing data management, tracing data lineage, and ensuring data quality." + }, + { + "name": "Saxo Bank", + "slug": "saxo-bank", + "imageUrl": "/img/logos/companies/saxobank.svg", + "imageSize": "default", + "link": "https://blog.datahubproject.io/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f", + "linkType": "blog", + "tagline": "Enabling Data Discovery in a Data Mesh", + "category": "Financial & Fintech", + "description": "Saxo Bank adopted DataHub to enhance data quality and streamline governance, facilitating efficient data management through self-service capabilities.

By integrating Apache Kafka and Snowflake with DataHub, the bank embraced Data Mesh principles to democratize data, support rapid growth, and improve business processes." + }, + { + "name": "MediaMarkt Saturn", + "slug": "mediamarkt-saturn", + "imageUrl": "/img/logos/companies/mediamarkt-saturn.png", + "imageSize": "large", + "link": "https://www.youtube.com/watch?v=wsCFnElN_Wo", + "linkType": "video", + "tagline": "DataHub + MediaMarktSaturn Access Management Journey", + "category": "B2B & B2C", + "description": "Europe’s #1 consumer electronics retailer implemented DataHub for three reasons:

1. DataHub provides an extremely flexible and customizable metadata platform at scale.
2. Open-source means lower cost to implement and removes the headache of license management.
3. Community-driven project which continually evolves with industry trends and best practices." + }, + { + "name": "Adevinta", + "slug": "adevinta", + "imageUrl": "/img/logos/companies/adevinta.png", + "imageSize": "medium", + "link": "https://medium.com/@adevinta/building-the-data-catalogue-the-beginning-of-a-journey-d64e828f955c", + "linkType": "blog", + "tagline": "Building the data catalogue", + "category": "E-Commerce", + "description": "“DataHub allows us to solve the data discovery problem, which was a big challenge in our organization, and now we are solving it.”" + }, + { + "name": "Wolt", + "slug": "wolt", + "imageUrl": "/img/logos/companies/wolt.png", + "imageSize": "default", + "link": "https://blog.datahubproject.io/humans-of-datahub-fredrik-sannholm-d673b1877f2b", + "linkType": "blog", + "tagline": "Wolt's DataHub Integration", + "category": "E-Commerce", + "description": "“[DataHub] has made our legal team very happy with being able to keep track of our sensitive data [to answer questions like] Where’s it going? How’s it being processed? Where’s it ending up? Which third party tool or API’s are we sending it to and why? Who is responsible for this integration?”" + }, + { + "name": "Geotab", + "slug": "geotab", + "imageUrl": "/img/logos/companies/geotab.jpg", + "imageSize": "small", + "link": "https://www.youtube.com/watch?v=boyjT2OrlU4", + "linkType": "video", + "tagline": "Geotab's Experience with DataHub", + "category": "B2B & B2C", + "description": "“The key evaluation metric for selecting DataHub was the approachability and technical capabilities of its leading development team.”

Geotab’s data adoption journey explores challenges in data management, governance, and the decision to utilize DataHub for improved productivity and collaboration." + }, + { + "name": "Hurb", + "slug": "hurb", + "imageUrl": "/img/logos/companies/hurb.png", + "imageSize": "medium", + "link": "https://blog.datahubproject.io/humans-of-datahub-patrick-franco-braz-b02b55a4c5384", + "linkType": "blog", + "tagline": "Hurb's DataHub Journey", + "category": "B2B & B2C", + "description": "“The main points that drove our decision to implement DataHub were its user-friendly interface, active and receptive community, contribution opportunities, and built-in ingestion sources for our primary services.”

Hurb implemented DataHub to enhance data governance, streamline data access, and improve decision-making through a structured integration process." + }, + { + "name": "Grofers", + "slug": "grofers", + "imageUrl": "/img/logos/companies/grofers.png", + "imageSize": "medium", + "link": "https://www.youtube.com/watch?v=m9kUYAuezFI", + "linkType": "video", + "tagline": "Grofers' Success with DataHub", + "category": "E-Commerce", + "description": "Grofers provides a closer look into how their team has leveraged DataHub as the source of truth for data governance." + }, + { + "name": "Viasat", + "slug": "viasat", + "imageUrl": "/img/logos/companies/viasat.png", + "imageSize": "medium", + "link": "https://www.youtube.com/watch?v=2SrDAJnzkjE", + "linkType": "video", + "tagline": "Viasat's DataHub Implementation", + "category": "And More", + "description": "Viasat highlights why they chose DataHub over other open source and commercial technologies and their plans with it." + }, + { + "name": "LinkedIn", + "slug": "linkedin", + "imageUrl": "/img/logos/companies/linkedin.svg", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Udemy", + "slug": "udemy", + "imageUrl": "/img/logos/companies/udemy.png", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "ThoughtWorks", + "slug": "thoughtworks", + "imageUrl": "/img/logos/companies/thoughtworks.png", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Expedia Group", + "slug": "expedia-group", + "imageUrl": "/img/logos/companies/expedia.svg", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Typeform", + "slug": "typeform", + "imageUrl": "/img/logos/companies/typeform.svg", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Peloton", + "slug": "peloton", + "imageUrl": "/img/logos/companies/peloton.png", + "imageSize": "default", + "category": "B2B & B2C" + }, + { + "name": "Razer", + "slug": "razer", + "imageUrl": "/img/logos/companies/razer.jpeg", + "imageSize": "large", + "category": "B2B & B2C" + }, + { + "name": "ClassDojo", + "slug": "classdojo", + "imageUrl": "/img/logos/companies/classdojo.png", + "imageSize": "medium", + "category": "B2B & B2C" + }, + { + "name": "Klarna", + "slug": "klarna", + "imageUrl": "/img/logos/companies/klarna.svg", + "imageSize": "medium", + "category": "Financial & Fintech" + }, + { + "name": "N26", + "slug": "n26", + "imageUrl": "/img/logos/companies/n26.svg", + "imageSize": "medium", + "category": "Financial & Fintech" + }, + { + "name": "BankSalad", + "slug": "banksalad", + "imageUrl": "/img/logos/companies/banksalad.png", + "imageSize": "default", + "category": "Financial & Fintech" + }, + { + "name": "Uphold", + "slug": "uphold", + "imageUrl": "/img/logos/companies/uphold.png", + "imageSize": "default", + "category": "Financial & Fintech" + }, + { + "name": "Stash", + "slug": "stash", + "imageUrl": "/img/logos/companies/stash.svg", + "imageSize": "medium", + "category": "Financial & Fintech" + }, + { + "name": "SumUp", + "slug": "sumup", + "imageUrl": "/img/logos/companies/sumup.png", + "imageSize": "medium", + "category": "Financial & Fintech" + }, + { + "name": "VanMoof", + "slug": "vanmoof", + "imageUrl": "/img/logos/companies/vanmoof.png", + "imageSize": "small", + "category": "E-Commerce" + }, + { + "name": "SpotHero", + "slug": "spothero", + "imageUrl": "/img/logos/companies/spothero.png", + "imageSize": "default", + "category": "E-Commerce" + }, + { + "name": "hipages", + "slug": "hipages", + "imageUrl": "/img/logos/companies/hipages.png", + "imageSize": "medium", + "category": "E-Commerce" + }, + { + "name": "Showroomprive.com", + "slug": "showroomprive-com", + "imageUrl": "/img/logos/companies/showroomprive.png", + "imageSize": "small", + "category": "E-Commerce" + }, + { + "name": "Wikimedia Foundation", + "slug": "wikimedia-foundation", + "imageUrl": "/img/logos/companies/wikimedia-foundation.png", + "imageSize": "medium", + "category": "And More" + }, + { + "name": "Cabify", + "slug": "cabify", + "imageUrl": "/img/logos/companies/cabify.png", + "imageSize": "medium", + "category": "And More" + }, + { + "name": "Digital Turbine", + "slug": "digital-turbine", + "imageUrl": "/img/logos/companies/digitalturbine.svg", + "imageSize": "medium", + "category": "And More" + }, + { + "name": "DFDS", + "slug": "dfds", + "imageUrl": "/img/logos/companies/dfds.png", + "imageSize": "medium", + "category": "And More" + }, + { + "name": "Moloco", + "slug": "moloco", + "imageUrl": "/img/logos/companies/moloco.png", + "imageSize": "medium", + "category": "And More" + } + ] +} \ No newline at end of file diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index ab033f7b04e4ba..1a40c986b31671 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -147,8 +147,8 @@ module.exports = { label: "YouTube", }, { - href: "https://www.youtube.com/playlist?list=PLdCtLs64vZvGCKMQC2dJEZ6cUqWsREbFi", - label: "Case Studies", + href: "/adoption-stories", + label: "Adoption Stories", }, { href: "https://www.youtube.com/playlist?list=PLdCtLs64vZvErAXMiqUYH9e63wyDaMBgg", diff --git a/docs-website/package.json b/docs-website/package.json index 62d12888323036..58820fbf42b21b 100644 --- a/docs-website/package.json +++ b/docs-website/package.json @@ -48,6 +48,7 @@ "react-dom": "18.2.0", "sass": "^1.43.2", "swc-loader": "^0.2.6", + "swiper": "^11.1.4", "uuid": "^9.0.0" }, "browserslist": { diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index d435f00902d771..75fc1f2dcd0c5d 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -31,7 +31,11 @@ module.exports = { label: "Demo", href: "https://demo.datahubproject.io/", }, - "docs/what-is-datahub/customer-stories", + { + type: "link", + label: "Adoption Stories", + href: "/adoption-stories", + }, "docs/what-is-datahub/datahub-concepts", ], }, diff --git a/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss b/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss deleted file mode 100644 index fcd3666d03ddc9..00000000000000 --- a/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss +++ /dev/null @@ -1,24 +0,0 @@ -.flexCol { - display: flex; -} - -.ctaCard { - flex-direction: row; - align-items: flex-start; - justify-content: space-between; - row-gap: 1rem; - padding: 1rem; - &:hover { - text-decoration: none; - border: 1px solid var(--ifm-color-primary); - background-color: var(--ifm-background-surface-color); - } - margin-bottom: 1rem; - flex: 1; -} - -.ctaHeading { - margin-bottom: 0; - display: flex; - align-items: center; -} diff --git a/docs-website/src/pages/_components/CardCTAs/index.js b/docs-website/src/pages/_components/CardCTAs/index.js deleted file mode 100644 index dc1b148d24bcd2..00000000000000 --- a/docs-website/src/pages/_components/CardCTAs/index.js +++ /dev/null @@ -1,52 +0,0 @@ -import React from "react"; -import clsx from "clsx"; -import styles from "./cardCTAs.module.scss"; -import useBaseUrl from "@docusaurus/useBaseUrl"; -import { ArrowRightOutlined } from "@ant-design/icons"; - -const cardsContent = [ - { - label: "Data Mesh", - title: "Data Products, Delivered", - url: "https://www.acryldata.io/blog/data-products-in-datahub-everything-you-need-to-know?utm_source=datahub&utm_medium=referral&utm_content=blog", - }, - { - label: "Data Contracts", - title: "Data Contracts: End-to-end Reliability in Data", - url: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management?utm_source=datahub&utm_medium=referral&utm_content=blog", - }, - { - label: "Shift Left", - title: "Data Governance and Lineage Impact Analysis", - url: "https://www.acryldata.io/blog/the-3-must-haves-of-metadata-management-part-2?utm_source=datahub&utm_medium=referral&utm_content=blog", - }, -]; - -const Card = ({ label, title, url }) => { - return ( - - ); -}; - -const CardCTAs = () => - cardsContent?.length > 0 ? ( -
-
-
- {cardsContent.map((props, idx) => ( - - ))} -
-
-
- ) : null; - -export default CardCTAs; diff --git a/docs-website/src/pages/_components/Hero/hero.module.scss b/docs-website/src/pages/_components/Hero/hero.module.scss index 6e4a623f469d51..97bdceaef69366 100644 --- a/docs-website/src/pages/_components/Hero/hero.module.scss +++ b/docs-website/src/pages/_components/Hero/hero.module.scss @@ -42,58 +42,3 @@ } } } - -.quickLinks { - display: flex; - align-items: center; - justify-content: space-between; - padding: 1rem; - font-weight: bold; - margin-bottom: -2.5vh; - @media (min-width: 768px) { - flex-direction: row; - } - - > * { - padding: 0.5rem 1rem; - display: inline-block; - - @media (min-width: 768px) { - padding: 0 1rem; - } - } -} - -.quickLinksLabel { - display: flex; - align-items: center; - svg { - width: 24px; - height: 24px; - color: var(--ifm-text-color) !important; - margin-right: 0.5rem; - } -} - -.quickstartContent { - text-align: center; - padding: 2rem 0; - height: 100%; - margin: 2rem 0; - background: #34394d; - border-radius: var(--ifm-card-border-radius); -} - -.quickstartTitle { - color: #fafafa; -} - -.quickstartSubtitle { - font-size: 1.1rem; - color: gray; -} - -.quickstartCodeblock { - text-align: left; - padding: 0 20vh; -} diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js index 17e5d0e7f4966a..12e41e2ecd1766 100644 --- a/docs-website/src/pages/_components/Hero/index.js +++ b/docs-website/src/pages/_components/Hero/index.js @@ -7,8 +7,8 @@ import { useColorMode } from "@docusaurus/theme-common"; import { QuestionCircleOutlined } from "@ant-design/icons"; import styles from "./hero.module.scss"; import CodeBlock from "@theme/CodeBlock"; -import CardCTAs from "../CardCTAs"; import TownhallButton from "../TownhallButton"; +import { Section } from "../Section"; const HeroAnnouncement = ({ message, linkUrl, linkText }) => (
@@ -50,33 +50,6 @@ const Hero = ({}) => {
- -
-

Get Started Now

-

Run the following command to get started with DataHub.

-
- - python3 -m pip install --upgrade pip wheel setuptools
- python3 -m pip install --upgrade acryl-datahub
- datahub docker quickstart - -
- - DataHub Quickstart Guide - - - Deploying With Kubernetes - -
-
-
- - Learn -
- What is DataHub? - How is DataHub architected? - See DataHub in action -
); diff --git a/docs-website/src/pages/_components/Logos/index.js b/docs-website/src/pages/_components/Logos/index.js index 3243617bcc40d6..565f6e9a46feec 100644 --- a/docs-website/src/pages/_components/Logos/index.js +++ b/docs-website/src/pages/_components/Logos/index.js @@ -1,204 +1,15 @@ -import React from "react"; import clsx from "clsx"; -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; import Link from "@docusaurus/Link"; import useBaseUrl from "@docusaurus/useBaseUrl"; - +import React from "react"; +import { Swiper, SwiperSlide } from "swiper/react"; +import "swiper/css"; +import "swiper/css/pagination"; +import { Pagination } from "swiper/modules"; import styles from "./logos.module.scss"; +const companyIndexes = require("../../../../adoptionStoriesIndexes.json"); +const companies = companyIndexes.companies; -const companiesByIndustry = [ - { - name: "B2B & B2C", - companies: [ - { - name: "LinkedIn", - imageUrl: "/img/logos/companies/linkedin.svg", - imageSize: "medium", - }, - { - name: "Udemy", - imageUrl: "/img/logos/companies/udemy.png", - imageSize: "medium", - }, - { - name: "Airtel", - imageUrl: "/img/logos/companies/airtel.png", - imageSize: "large", - }, - { - name: "Coursera", - imageUrl: "/img/logos/companies/coursera.svg", - imageSize: "small", - }, - { - name: "Geotab", - imageUrl: "/img/logos/companies/geotab.jpg", - imageSize: "small", - }, - { - name: "ThoughtWorks", - imageUrl: "/img/logos/companies/thoughtworks.png", - imageSize: "medium", - }, - { - name: "Expedia Group", - imageUrl: "/img/logos/companies/expedia.svg", - imageSize: "medium", - }, - { - name: "Typeform", - imageUrl: "/img/logos/companies/typeform.svg", - imageSize: "medium", - }, - { - name: "Peloton", - imageUrl: "/img/logos/companies/peloton.png", - imageSize: "default", - }, - { - name: "Zynga", - imageUrl: "/img/logos/companies/zynga.png", - imageSize: "default", - }, - { - name: "Hurb", - imageUrl: "/img/logos/companies/hurb.png", - imageSize: "medium", - }, - { - name: "Razer", - imageUrl: "/img/logos/companies/razer.jpeg", - imageSize: "large", - }, - { - name: "ClassDojo", - imageUrl: "/img/logos/companies/classdojo.png", - imageSize: "medium", - }, - ], - }, - { - name: "Financial & Fintech", - companies: [ - { - name: "Saxo Bank", - imageUrl: "/img/logos/companies/saxobank.svg", - imageSize: "default", - }, - { - name: "Klarna", - imageUrl: "/img/logos/companies/klarna.svg", - imageSize: "medium", - }, - { - name: "N26", - imageUrl: "/img/logos/companies/n26.svg", - imageSize: "medium", - }, - { - name: "BankSalad", - imageUrl: "/img/logos/companies/banksalad.png", - imageSize: "default", - }, - { - name: "Uphold", - imageUrl: "/img/logos/companies/uphold.png", - imageSize: "default", - }, - { - name: "Stash", - imageUrl: "/img/logos/companies/stash.svg", - imageSize: "medium", - }, - { - name: "SumUp", - imageUrl: "/img/logos/companies/sumup.png", - imageSize: "medium", - }, - ], - }, - { - name: "E-Commerce", - companies: [ - { - name: "Adevinta", - imageUrl: "/img/logos/companies/adevinta.png", - imageSize: "medium", - }, - { - name: "VanMoof", - imageUrl: "/img/logos/companies/vanmoof.png", - imageSize: "small", - }, - { - name: "Grofers", - imageUrl: "/img/logos/companies/grofers.png", - imageSize: "medium", - }, - { - name: "SpotHero", - imageUrl: "/img/logos/companies/spothero.png", - imageSize: "default", - }, - { - name: "hipages", - imageUrl: "/img/logos/companies/hipages.png", - imageSize: "medium", - }, - { - name: "Wolt", - imageUrl: "/img/logos/companies/wolt.png", - imageSize: "default", - }, - { - name: "Showroomprive.com", - imageUrl: "/img/logos/companies/showroomprive.png", - imageSize: "small", - }, - ], - }, - { - name: "And More", - companies: [ - { - name: "Wikimedia Foundation", - imageUrl: "/img/logos/companies/wikimedia-foundation.png", - imageSize: "medium", - }, - { - name: "Cabify", - imageUrl: "/img/logos/companies/cabify.png", - imageSize: "medium", - }, - { - name: "Digital Turbine", - imageUrl: "/img/logos/companies/digitalturbine.svg", - imageSize: "medium", - }, - { - name: "Viasat", - imageUrl: "/img/logos/companies/viasat.png", - imageSize: "medium", - }, - { - name: "DFDS", - imageUrl: "/img/logos/companies/dfds.png", - imageSize: "medium", - }, - { - name: "Moloco", - imageUrl: "/img/logos/companies/moloco.png", - imageSize: "medium", - }, - { - name: "Optum", - imageUrl: "/img/logos/companies/optum.jpg", - imageSize: "medium", - }, - ], - }, -]; const platformLogos = [ { @@ -315,10 +126,19 @@ const platformLogos = [ ]; export const PlatformLogos = () => ( - +
{[...platformLogos, ...platformLogos].map((logo, idx) => ( - {logo.name} + {logo.name} ))}
@@ -326,22 +146,58 @@ export const PlatformLogos = () => ( export const CompanyLogos = () => (
- - {companiesByIndustry.map((industry, idx) => ( - -
- {industry.companies.map((company, idx) => ( + + {companies.map((company, idx) => ( + + {company.link ? ( + {company.name} - ))} -
- + + ) : ( + {company.name} + )} + ))} - +
); diff --git a/docs-website/src/pages/_components/Logos/logos.module.scss b/docs-website/src/pages/_components/Logos/logos.module.scss index b20cc9a48b247c..fd331bccb45563 100644 --- a/docs-website/src/pages/_components/Logos/logos.module.scss +++ b/docs-website/src/pages/_components/Logos/logos.module.scss @@ -1,3 +1,7 @@ +.pillTabs { + justify-content: center; +} + .marquee { width: 100%; overflow: hidden; @@ -35,7 +39,6 @@ } .companyWrapper { - background: #fff; display: flex; flex-wrap: wrap; align-items: center; @@ -45,16 +48,25 @@ filter: invert(1); mix-blend-mode: exclusion; } + + :global { + .swiper-wrapper { + display: flex; + align-items: center; + margin-bottom: 1rem; + .swiper-slide { + display: flex; + justify-content: center; + align-items: center; + height: 100%; + } + } + } } .companyLogoContainer { - display: flex; - align-items: center; justify-content: center; > div { - display: flex; - flex-direction: column; - align-items: center; ul[role="tablist"] { padding: 0 1rem; overflow-x: auto; @@ -79,23 +91,35 @@ } } +.companyLogoWithLink { + &:hover { + opacity: 1; + filter: grayscale(0%); + } +} + .companyLogo { - width: auto; + flex-shrink: 0; + width: 100%; + height: auto; mix-blend-mode: luminosity; opacity: 0.66; - margin: 2.5rem; - height: 60px; + filter: grayscale(100%); &.default { - height: 60px; + padding: 30px; } &.large { - height: 100px; + padding: 5px; } &.medium { - height: 30px; + padding: 15px; } &.small { - height: 20px; + padding: 10px; } } + +.swiper-pagination { + margin-top: 1rem; +} \ No newline at end of file diff --git a/docs-website/src/pages/_components/QuickstartContent/index.js b/docs-website/src/pages/_components/QuickstartContent/index.js new file mode 100644 index 00000000000000..8c942a6a2e440b --- /dev/null +++ b/docs-website/src/pages/_components/QuickstartContent/index.js @@ -0,0 +1,50 @@ +import React from "react"; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import useBaseUrl from "@docusaurus/useBaseUrl"; +import Image from "@theme/IdealImage"; +import { useColorMode } from "@docusaurus/theme-common"; +import { QuestionCircleOutlined } from "@ant-design/icons"; +import styles from "./quickstartcontent.module.scss"; +import CodeBlock from "@theme/CodeBlock"; +import TownhallButton from "../TownhallButton"; +import { Section } from "../Section"; + + +const QuickstartContent = ({}) => { + const { colorMode } = useColorMode(); + return ( +
+
+
+

Get Started Now

+

Run the following command to get started with DataHub.

+
+ + python3 -m pip install --upgrade pip wheel setuptools
+ python3 -m pip install --upgrade acryl-datahub
+ datahub docker quickstart + +
+ + DataHub Quickstart Guide + + + Deploying With Kubernetes + +
+
+
+ + Learn +
+ What is DataHub? + How is DataHub architected? + See DataHub in action +
+
+
+ ); +}; + +export default QuickstartContent; diff --git a/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss new file mode 100644 index 00000000000000..e1badca6d2e348 --- /dev/null +++ b/docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss @@ -0,0 +1,67 @@ +.container { + margin-bottom: 2rem; +} + +.button { + text-decoration: none; + margin: 0.5rem 0 0 0; + white-space: nowrap; + @media (min-width: 690px) { + margin: 0 0 0 0.5rem; + } +} + +.quickstartContent { + text-align: center; + padding: 2rem 0; + height: 100%; + margin: 2rem 0; + background: #34394d; + border-radius: var(--ifm-card-border-radius); +} + +.quickstartTitle { + color: #fafafa; +} + +.quickstartSubtitle { + font-size: 1.1rem; + color: gray; +} + +.quickstartCodeblock { + text-align: left; + padding: 0 20vh; +} + +.quickLinks { + display: flex; + align-items: center; + justify-content: space-between; + padding: 1rem; + font-weight: bold; + margin-bottom: -2.5vh; + @media (min-width: 768px) { + flex-direction: row; + } + + > * { + padding: 0.5rem 1rem; + display: inline-block; + + @media (min-width: 768px) { + padding: 0 1rem; + } + } +} + +.quickLinksLabel { + display: flex; + align-items: center; + svg { + width: 24px; + height: 24px; + color: var(--ifm-text-color) !important; + margin-right: 0.5rem; + } +} diff --git a/docs-website/src/pages/_components/Quotes/index.js b/docs-website/src/pages/_components/Quotes/index.js index b66a04c2c6538e..664eba0b120519 100644 --- a/docs-website/src/pages/_components/Quotes/index.js +++ b/docs-website/src/pages/_components/Quotes/index.js @@ -32,7 +32,7 @@ const quotesContent = [ const Quote = ({ quote, company }) => { return ( -
+
{quote}
diff --git a/docs-website/src/pages/_components/Quotes/quotes.module.scss b/docs-website/src/pages/_components/Quotes/quotes.module.scss index 59573fa7a597a2..3bd895c4e3b243 100644 --- a/docs-website/src/pages/_components/Quotes/quotes.module.scss +++ b/docs-website/src/pages/_components/Quotes/quotes.module.scss @@ -8,6 +8,7 @@ .companyLogoWrapper { background: #fff; + height: 100px; html[data-theme="dark"] & { filter: invert(1); mix-blend-mode: exclusion; diff --git a/docs-website/src/pages/adoption-stories/_components/LearnItemCard/index.jsx b/docs-website/src/pages/adoption-stories/_components/LearnItemCard/index.jsx new file mode 100644 index 00000000000000..67b94788d97800 --- /dev/null +++ b/docs-website/src/pages/adoption-stories/_components/LearnItemCard/index.jsx @@ -0,0 +1,25 @@ +import React from "react"; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import styles from "./styles.module.scss"; + +const LearnItemCard = React.forwardRef(({ company, isSelected }, ref) => { + return ( +
+
+
+ {company.name} +
+
+
+
+ + Discover {company.name}'s Story + +
+
+
+ ); +}); + +export default LearnItemCard; diff --git a/docs-website/src/pages/adoption-stories/_components/LearnItemCard/styles.module.scss b/docs-website/src/pages/adoption-stories/_components/LearnItemCard/styles.module.scss new file mode 100644 index 00000000000000..881e90a7d09763 --- /dev/null +++ b/docs-website/src/pages/adoption-stories/_components/LearnItemCard/styles.module.scss @@ -0,0 +1,66 @@ +.featureCol { + display: flex; +} + +.card_date { + padding: 1rem 2rem; + font-size: 0.8rem; + font-style: italic; + color: gray; + margin-top: auto; +} + +.card_feature { + font-size: 2rem; + font-weight: 700; +} + +.card_button { + padding: 1rem; + text-align: center; +} + +.card { + color: var(--ifm-text-color); + text-decoration: none !important; + padding: 0rem; + margin-bottom: 2rem; + align-self: stretch; + flex-grow: 1; + &:hover { + opacity: 0.9; + } + + &.selected { + border-color: var(--ifm-color-primary); + box-shadow: rgba(0, 0, 0, 0.35) 0px 5px 15px; + scroll-margin-top: 100px; + } + + + hr { + margin: 0; + } +} + +.featureHeader { + h2 { + margin-bottom: 1rem !important; + font-size: 1.25rem; + } + padding: 1rem 2rem; +} + +.featureBody { + padding: 0 2rem; +} + +.card_image { + margin: 0; + margin-bottom: 0.5rem; + + img { + width: 100%; + height: auto; + } +} \ No newline at end of file diff --git a/docs-website/src/pages/adoption-stories/index.jsx b/docs-website/src/pages/adoption-stories/index.jsx new file mode 100644 index 00000000000000..27f4b876af20a6 --- /dev/null +++ b/docs-website/src/pages/adoption-stories/index.jsx @@ -0,0 +1,83 @@ +import React, { useState, useEffect, useRef } from "react"; +import Layout from "@theme/Layout"; +import BrowserOnly from "@docusaurus/BrowserOnly"; +import LearnItemCard from "./_components/LearnItemCard"; +import styles from "./styles.module.scss"; + +import customerStoriesIndexes from "../../../adoptionStoriesIndexes.json"; + +function AdoptionStoriesListPageContent() { + const companies = (customerStoriesIndexes?.companies || []).filter((company) => company.link); + const [activeFilters, setActiveFilters] = useState([]); + const categories = ["B2B & B2C", "E-Commerce", "Financial & Fintech", "And More"]; + const selectedCardRef = useRef(null); + + const filteredItems = activeFilters.length + ? companies.filter((company) => activeFilters.includes(company.category)) + : companies; + + const handleFilterToggle = (category) => { + if (activeFilters.includes(category)) { + setActiveFilters(activeFilters.filter((filter) => filter !== category)); + } else { + setActiveFilters([...new Set([...activeFilters, category])]); + } + }; + + useEffect(() => { + const selectedSlug = window.location.hash.substring(1); + if (selectedCardRef.current) { + selectedCardRef.current.scrollIntoView({ behavior: "smooth", block: "start", inline: "nearest" }); + } + }, [selectedCardRef]); + + return ( + +
+
+
+
+

DataHub Adoption Stories

+

Learn how the best data and AI teams are using DataHub +
+ Check out more stories on the DataHub Youtube. +

+
+
+
+ For: + {categories.map((category) => ( + + ))} +
+
+
+
+
+ {filteredItems.map((company) => ( + + ))} +
+
+ + ); +} + +export default function AdoptionStoriesListPage() { + return ( + + {() => } + + ); +} diff --git a/docs-website/src/pages/adoption-stories/styles.module.scss b/docs-website/src/pages/adoption-stories/styles.module.scss new file mode 100644 index 00000000000000..d08b48a011de07 --- /dev/null +++ b/docs-website/src/pages/adoption-stories/styles.module.scss @@ -0,0 +1,7 @@ +.filterBar { + display: flex; + justify-content: center; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} \ No newline at end of file diff --git a/docs-website/src/pages/cloud/CompanyLogos/customersData.json b/docs-website/src/pages/cloud/CompanyLogos/customersData.json index 09943e8ef172fc..e8a7470eab4702 100644 --- a/docs-website/src/pages/cloud/CompanyLogos/customersData.json +++ b/docs-website/src/pages/cloud/CompanyLogos/customersData.json @@ -2,38 +2,50 @@ "customers": [ { "link": { - "href": "https://robinhood.com", + "href": "https://www.depop.com/", "blank": true }, "logo": { "asset": { - "_ref": "/img/logos/scrollingCompanies/acertus.webp" + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/345d124a249d43bf0f20b608f8bfa2f7683311fa-360x180.png" }, - "alt": "Robinhood" + "alt": "depop" } }, { "link": { - "href": "https://www.dpgmediagroup.com/", + "href": "https://riskified.com", "blank": true }, "logo": { "asset": { - "_ref": "/img/logos/scrollingCompanies/autoscout24.webp" + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/c982e0459bed565273a9b696d9d40aed76f84b1e-360x180.png" }, - "alt": "DPG Media" + "alt": "Riskified" } }, { "link": { - "href": "https://www.twilio.com", + "href": "https://get.betterup.com/", "blank": true }, "logo": { "asset": { - "_ref": "/img/logos/scrollingCompanies/betterup.webp" + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/5988a55b3c090a12ddc3f3cae07b290ac3134771-360x180.png" }, - "alt": "Twilio" + "alt": "Betterup" + } + }, + { + "link": { + "href": "https://www.ovoenergy.com/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/5e7bd32dfbc769849dca136947ebd2fc2f5e91f3-540x270.png" + }, + "alt": "OVO Energy" } }, { @@ -43,45 +55,69 @@ }, "logo": { "asset": { - "_ref": "/img/logos/scrollingCompanies/depop.webp" + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/3d7c10e1bd7c7a062250e092d6d9d0553fb57790-360x180.png" }, "alt": "Myob" } }, { "link": { - "href": "https://regeneron.com", + "href": "https://www.dpgmediagroup.com/", "blank": true }, "logo": { "asset": { - "_ref": "/img/logos/scrollingCompanies/dpg_media.webp" + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/b446f595b4b13a72ee82a285924715f950e012ca-540x270.png" }, - "alt": "Regeneron" + "alt": "DPG Megia" } }, { "link": { - "href": "https://riskified.com", + "href": "https://www.notion.so/", "blank": true }, "logo": { "asset": { - "_ref": "/img/logos/scrollingCompanies/myob.webp" + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/c2e84f93572cd1baf30ea7ab8da234ff44182eb6-540x270.png" }, - "alt": "Riskified" + "alt": "Notion" + } + }, + { + "link": { + "href": "https://www.sae.org/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/a9e8586635cb4039cbfc5836a6a5cacdeba9e6b3-540x270.png" + }, + "alt": "SAE International" + } + }, + { + "link": { + "href": "https://viator.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/0c17141cad5baa053da18dffb17d9c182d242e69-1200x475.png" + }, + "alt": "Viator" } }, { "link": { - "href": "https://xero.com", + "href": "https://www.tripadvisor.co.uk/", "blank": true }, "logo": { "asset": { - "_ref": "/img/logos/scrollingCompanies/notion.webp" + "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/28255a28d261a074a83d1ee8632f0338bf5cf57e-1112x256.png" }, - "alt": "Xero" + "alt": "Trip Advisor" } } ] diff --git a/docs-website/src/pages/cloud/CompanyLogos/index.js b/docs-website/src/pages/cloud/CompanyLogos/index.js index cd3a79f277893a..d395d5b50e48ac 100644 --- a/docs-website/src/pages/cloud/CompanyLogos/index.js +++ b/docs-website/src/pages/cloud/CompanyLogos/index.js @@ -32,7 +32,7 @@ const ScrollingCustomers = ({ noOverlay = true, spacing, ...rest }) => { to={customer.link.href} target={customer.link.blank ? '_blank' : '_self'} rel={customer.link.blank ? 'noopener noreferrer' : ''} - style={{ minWidth: 'max-content', padding: '0 3.25rem' }} + style={{ minWidth: 'max-content', padding: '0 1.8rem' }} > Introducing DataHub as a Managed Service
with Data Observability and Data Governance built-in.
- */}
Book Demo diff --git a/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss b/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss deleted file mode 100644 index 349f705d25b10d..00000000000000 --- a/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss +++ /dev/null @@ -1,56 +0,0 @@ -.card { - color: var(--ifm-hero-text-color); - padding: 0; - margin: 0rem 3rem 2rem 0rem; - text-decoration: none !important; - - .card_button { - padding: 0rem 0rem 0rem 1rem; - text-align: right; - } - - .card_img { - justify-content: center; - display: flex; - height: 250px; - margin: 0; - position: relative; - text-align: center; - } - - .card_body { - padding: 2rem 3rem 2rem 3rem; - - .card_description { - min-height: 20rem; - } - } - - .card_overlay_text { - position: absolute; - text-align: left; - width: 80%; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); - color: white; - - .card_customer { - font-size: 3.2rem; - font-weight: 800; - line-height: 1.2; - - } - .card_title { - font-size: 1.2rem; - font-weight: 600; - } - - } - - img { - object-fit: cover; - filter: brightness(50%); - } - -} diff --git a/docs-website/src/pages/docs/_components/CustomerCard/index.jsx b/docs-website/src/pages/docs/_components/CustomerCard/index.jsx deleted file mode 100644 index 36c83226e1f732..00000000000000 --- a/docs-website/src/pages/docs/_components/CustomerCard/index.jsx +++ /dev/null @@ -1,30 +0,0 @@ -import React from "react"; -import clsx from "clsx"; -import styles from "./customercard.module.scss"; -import Link from "@docusaurus/Link"; - -const CustomerCard = ({ customer, title, imgUrl, description, to,}) => { - return ( -
-
-
- {customer} -
-
{customer}
-
{title}
-
-
-
-

{description}

-
- - Discover {customer}'s Story - -
-
-
-
- ); -}; - -export default CustomerCard; diff --git a/docs-website/src/pages/docs/_components/CustomerCardSection/customercardsection.module.scss b/docs-website/src/pages/docs/_components/CustomerCardSection/customercardsection.module.scss deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx b/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx index 27067cb3930ebc..505a2810c9433c 100644 --- a/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx +++ b/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx @@ -93,4 +93,4 @@ const CustomerCardSection = () => { ); }; -export default CustomerCardSection; +export default CustomerCardSection; \ No newline at end of file diff --git a/docs-website/src/pages/index.js b/docs-website/src/pages/index.js index 68b177d10f7aff..2eed41b4ad1bd3 100644 --- a/docs-website/src/pages/index.js +++ b/docs-website/src/pages/index.js @@ -3,13 +3,14 @@ import Layout from "@theme/Layout"; import Link from "@docusaurus/Link"; import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; import CodeBlock from "@theme/CodeBlock"; - +import useBaseUrl from "@docusaurus/useBaseUrl"; import Hero from "./_components/Hero"; import Features from "./_components/Features"; -import Quotes from "./_components/Quotes"; import { Section, PromoSection } from "./_components/Section"; -import { PlatformLogos, CompanyLogos } from "./_components/Logos"; +import { PlatformLogos } from "./_components/Logos"; import RoundedImage from "./_components/RoundedImage"; +import { CompanyLogos } from "./_components/Logos"; +import QuickstartContent from "./_components/QuickstartContent"; const example_recipe = ` source: @@ -38,6 +39,18 @@ function Home() { description="DataHub is a data discovery application built on an extensible data catalog that helps you tame the complexity of diverse data ecosystems." > +
+ +
+ + Check Out Adoption Stories → + +
+
+
@@ -157,10 +170,6 @@ function Home() {
-
- - -
) : null; } diff --git a/docs-website/static/img/adoption-stories/adoption-stories-adevinta.png b/docs-website/static/img/adoption-stories/adoption-stories-adevinta.png new file mode 100644 index 00000000000000..6c790995843c54 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-adevinta.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-airtel.png b/docs-website/static/img/adoption-stories/adoption-stories-airtel.png new file mode 100644 index 00000000000000..ae5ebdedd47aa1 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-airtel.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-coursera.png b/docs-website/static/img/adoption-stories/adoption-stories-coursera.png new file mode 100644 index 00000000000000..4f473874d0dc26 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-coursera.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-geotab.png b/docs-website/static/img/adoption-stories/adoption-stories-geotab.png new file mode 100644 index 00000000000000..2b3c8a158273a9 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-geotab.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-grofers.png b/docs-website/static/img/adoption-stories/adoption-stories-grofers.png new file mode 100644 index 00000000000000..51af8a3ad69d7b Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-grofers.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-hurb.png b/docs-website/static/img/adoption-stories/adoption-stories-hurb.png new file mode 100644 index 00000000000000..b7b8bae5d8c321 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-hurb.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-mediamarkt-saturn.png b/docs-website/static/img/adoption-stories/adoption-stories-mediamarkt-saturn.png new file mode 100644 index 00000000000000..ac2f524a7a0e77 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-mediamarkt-saturn.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-netflix.png b/docs-website/static/img/adoption-stories/adoption-stories-netflix.png new file mode 100644 index 00000000000000..de65a4c59419b5 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-netflix.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-optum.png b/docs-website/static/img/adoption-stories/adoption-stories-optum.png new file mode 100644 index 00000000000000..051abaa96a0e01 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-optum.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-pinterest.png b/docs-website/static/img/adoption-stories/adoption-stories-pinterest.png new file mode 100644 index 00000000000000..e005ea6d5750aa Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-pinterest.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-saxo-bank.png b/docs-website/static/img/adoption-stories/adoption-stories-saxo-bank.png new file mode 100644 index 00000000000000..333003d146cf5e Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-saxo-bank.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-viasat.png b/docs-website/static/img/adoption-stories/adoption-stories-viasat.png new file mode 100644 index 00000000000000..b6f633450296c6 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-viasat.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-visa.png b/docs-website/static/img/adoption-stories/adoption-stories-visa.png new file mode 100644 index 00000000000000..11d732faf85fec Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-visa.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-wolt.png b/docs-website/static/img/adoption-stories/adoption-stories-wolt.png new file mode 100644 index 00000000000000..43501a1f2f6d57 Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-wolt.png differ diff --git a/docs-website/static/img/adoption-stories/adoption-stories-zynga.png b/docs-website/static/img/adoption-stories/adoption-stories-zynga.png new file mode 100644 index 00000000000000..94ee9e9b2fb8ee Binary files /dev/null and b/docs-website/static/img/adoption-stories/adoption-stories-zynga.png differ diff --git a/docs-website/static/img/adoption-stories/img.png b/docs-website/static/img/adoption-stories/img.png new file mode 100644 index 00000000000000..4d4971018c3982 Binary files /dev/null and b/docs-website/static/img/adoption-stories/img.png differ diff --git a/docs-website/static/img/assets/business.jpg b/docs-website/static/img/assets/business.jpg deleted file mode 100644 index f5a91928ee2ad8..00000000000000 Binary files a/docs-website/static/img/assets/business.jpg and /dev/null differ diff --git a/docs-website/static/img/assets/netflix.jpg b/docs-website/static/img/assets/netflix.jpg deleted file mode 100644 index 8b555f5b63187f..00000000000000 Binary files a/docs-website/static/img/assets/netflix.jpg and /dev/null differ diff --git a/docs-website/static/img/assets/phonecall.jpg b/docs-website/static/img/assets/phonecall.jpg deleted file mode 100644 index 87e48f28213827..00000000000000 Binary files a/docs-website/static/img/assets/phonecall.jpg and /dev/null differ diff --git a/docs-website/static/img/assets/travel.jpg b/docs-website/static/img/assets/travel.jpg deleted file mode 100644 index de2697f5631217..00000000000000 Binary files a/docs-website/static/img/assets/travel.jpg and /dev/null differ diff --git a/docs-website/static/img/logos/companies/pinterest.png b/docs-website/static/img/logos/companies/pinterest.png new file mode 100644 index 00000000000000..715c8c33fd85b4 Binary files /dev/null and b/docs-website/static/img/logos/companies/pinterest.png differ diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index a93b0e74c327db..0970a59cbc00a3 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -1827,7 +1827,7 @@ "@docusaurus/theme-search-algolia" "2.4.3" "@docusaurus/types" "2.4.3" -"@docusaurus/react-loadable@5.5.2": +"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2": version "5.5.2" resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -9705,14 +9705,6 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1: dependencies: "@babel/runtime" "^7.10.3" -"react-loadable@npm:@docusaurus/react-loadable@5.5.2": - version "5.5.2" - resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" - integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== - dependencies: - "@types/react" "*" - prop-types "^15.6.2" - react-markdown@^8.0.6: version "8.0.7" resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b" @@ -10866,6 +10858,11 @@ swc-loader@^0.2.6: dependencies: "@swc/counter" "^0.1.3" +swiper@^11.1.4: + version "11.1.4" + resolved "https://registry.yarnpkg.com/swiper/-/swiper-11.1.4.tgz#2f8e303e8bf9e5bc40a3885fc637ae60ff27996c" + integrity sha512-1n7kbYJB2dFEpUHRFszq7gys/ofIBrMNibwTiMvPHwneKND/t9kImnHt6CfGPScMHgI+dWMbGTycCKGMoOO1KA== + symbol-observable@^1.0.4: version "1.2.0" resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804" diff --git a/docs/actions/guides/developing-a-transformer.md b/docs/actions/guides/developing-a-transformer.md index a843dbc846cd51..6406cdfae6104b 100644 --- a/docs/actions/guides/developing-a-transformer.md +++ b/docs/actions/guides/developing-a-transformer.md @@ -23,7 +23,7 @@ print the configuration that is provided when it is created, and print any Event ```python # custom_transformer.py from datahub_actions.transform.transformer import Transformer -from datahub_actions.event.event import EventEnvelope +from datahub_actions.event.event_envelope import EventEnvelope from datahub_actions.pipeline.pipeline_context import PipelineContext from typing import Optional @@ -75,7 +75,7 @@ Next, install the package pip install -e . ``` -inside the module. (alt.`python setup.py`). +inside the module. (alt.`python setup.py`). Once we have done this, our class will be referencable via `custom_transformer_example.custom_transformer:CustomTransformer`. @@ -96,7 +96,7 @@ source: connection: bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} -transform: +transform: - type: "custom_transformer_example.custom_transformer:CustomTransformer" config: # Some sample configuration which should be printed on create. @@ -130,4 +130,4 @@ it without defining the full module path. Prerequisites to consideration for inclusion in the core Transformer library include - **Testing** Define unit tests for your Transformer -- **Deduplication** Confirm that no existing Transformer serves the same purpose, or can be easily extended to serve the same purpose \ No newline at end of file +- **Deduplication** Confirm that no existing Transformer serves the same purpose, or can be easily extended to serve the same purpose diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index d1003077e24861..67dd9a734e67f5 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -76,7 +76,7 @@ First, if you did not use eksctl to setup the kubernetes cluster, make sure to g Download the IAM policy document for allowing the controller to make calls to AWS APIs on your behalf. ``` -curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.2.0/docs/install/iam_policy.json +curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json ``` Create an IAM policy based on the policy document by running the following. @@ -148,13 +148,9 @@ datahub-frontend: alb.ingress.kubernetes.io/certificate-arn: <> alb.ingress.kubernetes.io/inbound-cidrs: 0.0.0.0/0 alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/ssl-redirect: '443' hosts: - host: <> - redirectPaths: - - path: /* - name: ssl-redirect - port: use-annotation paths: - /* ``` diff --git a/docs/how/search.md b/docs/how/search.md index 6befb0a1e54f2d..c809ab1efba12d 100644 --- a/docs/how/search.md +++ b/docs/how/search.md @@ -148,24 +148,42 @@ The same GraphQL API that powers the Search UI can be used for integrations and programmatic use-cases. ``` -# Example query -{ - searchAcrossEntities( - input: {types: [], query: "*", start: 0, count: 10, filters: [{field: "fieldTags", value: "urn:li:tag:Dimension"}]} +# Example query - search for datasets matching the example_query_text who have the Dimension tag applied to a schema field and are from the data platform looker +query searchEntities { + search( + input: { + type: DATASET, + query: "example_query_text", + orFilters: [ + { + and: [ + { + field: "fieldTags", + values: ["urn:li:tag:Dimension"] + }, + { + field: "platform", + values: ["urn:li:dataPlatform:looker"] + } + ] + } + ], + start: 0, + count: 10 + } ) { start count total searchResults { entity { + urn type ... on Dataset { - urn - type + name platform { name } - name } } } diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 2821b63e7d305a..08ababcb5cfce9 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -81,7 +81,8 @@ profiling: - #10498 - Tableau ingestion can now be configured to ingest multiple sites at once and add the sites as containers. The feature is currently only available for Tableau Server. - #10466 - Extends configuration in `~/.datahubenv` to match `DatahubClientConfig` object definition. See full configuration in https://datahubproject.io/docs/python-sdk/clients/. The CLI should now respect the updated configurations specified in `~/.datahubenv` across its functions and utilities. This means that for systems where ssl certification is disabled, setting `disable_ssl_verification: true` in `~./datahubenv` will apply to all CLI calls. - #11002 - We will not auto-generate a `~/.datahubenv` file. You must either run `datahub init` to create that file, or set environment variables so that the config is loaded. - +- #11023 - Added a new parameter to datahub's `put` cli command: `--run-id`. This parameter is useful to associate a given write to an ingestion process. A use-case can be mimick transformers when a transformer for aspect being written does not exist. +- #11051 - Ingestion reports will now trim the summary text to a maximum of 800k characters to avoid generating `dataHubExecutionRequestResult` that are too large for GMS to handle. ## 0.13.3 ### Breaking Changes diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md index 9d838ef8a44042..2d7707637e2d1c 100644 --- a/docs/lineage/airflow.md +++ b/docs/lineage/airflow.md @@ -17,7 +17,7 @@ There's two actively supported implementations of the plugin, with different Air | Approach | Airflow Version | Notes | | --------- | --------------- | --------------------------------------------------------------------------- | -| Plugin v2 | 2.3+ | Recommended. Requires Python 3.8+ | +| Plugin v2 | 2.3.4+ | Recommended. Requires Python 3.8+ | | Plugin v1 | 2.1+ | No automatic lineage extraction; may not extract lineage if the task fails. | If you're using Airflow older than 2.1, it's possible to use the v1 plugin with older versions of `acryl-datahub-airflow-plugin`. See the [compatibility section](#compatibility) for more details. @@ -66,7 +66,7 @@ enabled = True # default ``` | Name | Default value | Description | -|----------------------------|----------------------|------------------------------------------------------------------------------------------| +| -------------------------- | -------------------- | ---------------------------------------------------------------------------------------- | | enabled | true | If the plugin should be enabled. | | conn_id | datahub_rest_default | The name of the datahub rest connection. | | cluster | prod | name of the airflow cluster, this is equivalent to the `env` of the instance | @@ -132,7 +132,7 @@ conn_id = datahub_rest_default # or datahub_kafka_default ``` | Name | Default value | Description | -|----------------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -------------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | enabled | true | If the plugin should be enabled. | | conn_id | datahub_rest_default | The name of the datahub connection you set in step 1. | | cluster | prod | name of the airflow cluster | @@ -240,6 +240,7 @@ See this [example PR](https://github.com/datahub-project/datahub/pull/10452) whi There might be a case where the DAGs are removed from the Airflow but the corresponding pipelines and tasks are still there in the Datahub, let's call such pipelines ans tasks, `obsolete pipelines and tasks` Following are the steps to cleanup them from the datahub: + - create a DAG named `Datahub_Cleanup`, i.e. ```python @@ -263,8 +264,8 @@ with DAG( ) ``` -- ingest this DAG, and it will remove all the obsolete pipelines and tasks from the Datahub based on the `cluster` value set in the `airflow.cfg` +- ingest this DAG, and it will remove all the obsolete pipelines and tasks from the Datahub based on the `cluster` value set in the `airflow.cfg` ## Get all dataJobs associated with a dataFlow @@ -274,12 +275,7 @@ If you are looking to find all tasks (aka DataJobs) that belong to a specific pi query { dataFlow(urn: "urn:li:dataFlow:(airflow,db_etl,prod)") { childJobs: relationships( - input: { - types: ["IsPartOf"], - direction: INCOMING, - start: 0, - count: 100 - } + input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 100 } ) { total relationships { diff --git a/docs/quick-ingestion-guides/tableau/setup.md b/docs/quick-ingestion-guides/tableau/setup.md index b6ccaf2a9cc9e5..81767215d5bcd5 100644 --- a/docs/quick-ingestion-guides/tableau/setup.md +++ b/docs/quick-ingestion-guides/tableau/setup.md @@ -51,6 +51,14 @@ In order to configure ingestion from Tableau, you'll first have to enable Tablea - Open a command prompt as an admin on the initial node (*where TSM is installed*) in the cluster - Run the command: `tsm maintenance metadata-services enable` +3. **Enable Derived Permissions:** This step is required only when the site is using external assets. For more detail, refer to the tableau documentation [Manage Permissions for External Assets](https://help.tableau.com/current/online/en-us/dm_perms_assets.htm). + + Follow the below steps to enable the derived permissions: + + - Sign in to Tableau Cloud or Tableau Server as an admin. + - From the left navigation pane, click Settings. + - On the General tab, under Automatic Access to Metadata about Databases and Tables, select the `Automatically grant authorized users access to metadata about databases and tables` check box. + ## Next Steps diff --git a/docs/what-is-datahub/customer-stories.md b/docs/what-is-datahub/customer-stories.md deleted file mode 100644 index 2745a8aca0d2f2..00000000000000 --- a/docs/what-is-datahub/customer-stories.md +++ /dev/null @@ -1,17 +0,0 @@ -import CustomerCardSection from '@site/src/pages/docs/_components/CustomerCardSection'; - -# Customer Stories - -Meet the DataHub users who have shared their stories with us. - -:::note Share Your DataHub Journey -We're excited to hear about your experience with DataHub. Share your story with us! - -Share Your Story - -::: - - diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/models/graph/Edge.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/models/graph/Edge.java index ca5e6d01103848..3de09e599d99ed 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/models/graph/Edge.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/models/graph/Edge.java @@ -72,8 +72,9 @@ public String toDocId() { } try { + String hashAlgo = System.getenv("ELASTIC_ID_HASH_ALGO"); byte[] bytesOfRawDocID = rawDocId.toString().getBytes(StandardCharsets.UTF_8); - MessageDigest md = MessageDigest.getInstance("MD5"); + MessageDigest md = MessageDigest.getInstance(hashAlgo); byte[] thedigest = md.digest(bytesOfRawDocID); return Base64.getEncoder().encodeToString(thedigest); } catch (NoSuchAlgorithmException e) { diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java index 899f66e66ea5ae..bef7782d8f7c9a 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java @@ -63,7 +63,7 @@ public static Map> extractFields( } else { List valueList = (List) value.get(); // If the field is a nested list of values, flatten it - for (int i = 0; i < numArrayWildcards - 1; i++) { + for (long i = 0; i < numArrayWildcards - 1; i++) { valueList = valueList.stream() .flatMap(v -> ((List) v).stream()) diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py index 6d5aa74b1d96ff..2401b169cd6607 100644 --- a/metadata-ingestion-modules/airflow-plugin/setup.py +++ b/metadata-ingestion-modules/airflow-plugin/setup.py @@ -53,7 +53,7 @@ def get_long_description(): mypy_stubs = { "types-dataclasses", "sqlalchemy-stubs", - "types-pkg_resources", + "types-setuptools", "types-six", "types-python-dateutil", "types-requests", diff --git a/metadata-ingestion-modules/dagster-plugin/setup.py b/metadata-ingestion-modules/dagster-plugin/setup.py index 60b960e653eb2b..8a2a1d76d345bf 100644 --- a/metadata-ingestion-modules/dagster-plugin/setup.py +++ b/metadata-ingestion-modules/dagster-plugin/setup.py @@ -26,14 +26,14 @@ def get_long_description(): "dagit >= 1.3.3", *rest_common, # Ignoring the dependency below because it causes issues with the vercel built wheel install - #f"acryl-datahub[datahub-rest]{_self_pin}", + # f"acryl-datahub[datahub-rest]{_self_pin}", "acryl-datahub[datahub-rest]", } mypy_stubs = { "types-dataclasses", "sqlalchemy-stubs", - "types-pkg_resources", + "types-setuptools", "types-six", "types-python-dateutil", "types-requests", diff --git a/metadata-ingestion/docs/sources/tableau/tableau_pre.md b/metadata-ingestion/docs/sources/tableau/tableau_pre.md index 5e323da6746d21..aeb67f85b241b9 100644 --- a/metadata-ingestion/docs/sources/tableau/tableau_pre.md +++ b/metadata-ingestion/docs/sources/tableau/tableau_pre.md @@ -81,3 +81,12 @@ This may happen when the Tableau API returns NODE_LIMIT_EXCEEDED error in respon - reducing the page size using the `page_size` config param in datahub recipe (Defaults to 10). - increasing tableau configuration [metadata query node limit](https://help.tableau.com/current/server/en-us/cli_configuration-set_tsm.htm#metadata_nodelimit) to higher value. + +### `PERMISSIONS_MODE_SWITCHED` error in ingestion report +This error occurs if the Tableau site is using external assets. For more detail, refer to the Tableau documentation [Manage Permissions for External Assets](https://help.tableau.com/current/online/en-us/dm_perms_assets.htm). + +Follow the below steps to enable the derived permissions: + +1. Sign in to Tableau Cloud or Tableau Server as an admin. +2. From the left navigation pane, click Settings. +3. On the General tab, under Automatic Access to Metadata about Databases and Tables, select the `Automatically grant authorized users access to metadata about databases and tables` check box. diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index fdd469a1268d44..d44a0a5c316f39 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -454,7 +454,7 @@ }, # FIXME: I don't think tableau uses sqllineage anymore so we should be able # to remove that dependency. - "tableau": {"tableauserverclient>=0.17.0"} | sqllineage_lib | sqlglot_lib, + "tableau": {"tableauserverclient>=0.24.0"} | sqllineage_lib | sqlglot_lib, "teradata": sql_common | usage_common | sqlglot_lib @@ -491,7 +491,7 @@ mypy_stubs = { "types-dataclasses", - "types-pkg_resources", + "types-setuptools", "types-six", "types-python-dateutil", # We need to avoid 2.31.0.5 and 2.31.0.4 due to diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py index 21841b173c23d3..1b9cccb1cbc215 100644 --- a/metadata-ingestion/src/datahub/cli/cli_utils.py +++ b/metadata-ingestion/src/datahub/cli/cli_utils.py @@ -1,5 +1,6 @@ import json import logging +import time import typing from datetime import datetime from typing import Any, Dict, List, Optional, Tuple, Type, Union @@ -403,6 +404,8 @@ def ensure_has_system_metadata( if event.systemMetadata is None: event.systemMetadata = SystemMetadataClass() metadata = event.systemMetadata + if metadata.lastObserved == 0: + metadata.lastObserved = int(time.time() * 1000) if metadata.properties is None: metadata.properties = {} props = metadata.properties diff --git a/metadata-ingestion/src/datahub/cli/get_cli.py b/metadata-ingestion/src/datahub/cli/get_cli.py index b6ff5f39a2c14b..27fa987ac79779 100644 --- a/metadata-ingestion/src/datahub/cli/get_cli.py +++ b/metadata-ingestion/src/datahub/cli/get_cli.py @@ -56,6 +56,7 @@ def urn(ctx: Any, urn: Optional[str], aspect: List[str], details: bool) -> None: entity_urn=urn, aspects=aspect, typed=False, + details=details, ), sort_keys=True, indent=2, diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index 29e1f63dd452ef..78a091f1ffe689 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -1,3 +1,4 @@ +import logging from typing import Iterable, Optional from pydantic.fields import Field @@ -18,6 +19,8 @@ from datahub.specific.dashboard import DashboardPatchBuilder from datahub.specific.dataset import DatasetPatchBuilder +logger = logging.getLogger(__name__) + def convert_upstream_lineage_to_patch( urn: str, @@ -48,6 +51,20 @@ def convert_chart_info_to_patch( for inputEdge in aspect.inputEdges: patch_builder.add_input_edge(inputEdge) + patch_builder.set_chart_url(aspect.chartUrl).set_external_url( + aspect.externalUrl + ).set_type(aspect.type).set_title(aspect.title).set_access( + aspect.access + ).set_last_modified( + aspect.lastModified + ).set_last_refreshed( + aspect.lastRefreshed + ).set_description( + aspect.description + ).add_inputs( + aspect.inputs + ) + values = patch_builder.build() if values: mcp = next(iter(values)) @@ -76,8 +93,36 @@ def convert_dashboard_info_to_patch( for chartEdge in aspect.chartEdges: patch_builder.add_chart_edge(chartEdge) + if aspect.title: + patch_builder.set_title(aspect.title) + + if aspect.description: + patch_builder.set_description(aspect.description) + + if aspect.charts: + patch_builder.add_charts(aspect.charts) + + if aspect.dashboardUrl: + patch_builder.set_dashboard_url(aspect.dashboardUrl) + + if aspect.datasets: + patch_builder.add_datasets(aspect.datasets) + + if aspect.access: + patch_builder.set_access(aspect.access) + + if aspect.lastRefreshed: + patch_builder.set_last_refreshed(aspect.lastRefreshed) + + if aspect.lastModified: + patch_builder.set_last_modified(last_modified=aspect.lastModified) + values = patch_builder.build() + if values: + logger.debug( + f"Generating patch DashboardInfo MetadataWorkUnit for dashboard {aspect.title}" + ) mcp = next(iter(values)) return MetadataWorkUnit( id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py index 788bec97a64884..a4de8b382430c7 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source.py @@ -137,7 +137,10 @@ def report_log( ) # Add the simple exception details to the context. - context = f"{context}: {exc}" + if context: + context = f"{context} {type(exc)}: {exc}" + else: + context = f"{type(exc)}: {exc}" elif log: logger.log(level=level.value, msg=log_content, stacklevel=stacklevel) diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index 4172c89fbfacff..55bd0b3cf0afc1 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -501,8 +501,8 @@ def get_aspects_for_entity( responds to these calls, and will be fixed automatically when the server-side issue is fixed. :param str entity_urn: The urn of the entity - :param List[Type[Aspect]] aspect_type_list: List of aspect type classes being requested (e.g. [datahub.metadata.schema_classes.DatasetProperties]) - :param List[str] aspects_list: List of aspect names being requested (e.g. [schemaMetadata, datasetProperties]) + :param aspects: List of aspect names being requested (e.g. [schemaMetadata, datasetProperties]) + :param aspect_types: List of aspect type classes being requested (e.g. [datahub.metadata.schema_classes.DatasetProperties]) :return: Optionally, a map of aspect_name to aspect_value as a dictionary if present, aspect_value will be set to None if that aspect was not found. Returns None on HTTP status 404. :raises HttpError: if the HTTP response is not a 200 """ @@ -527,8 +527,10 @@ def get_aspects_for_entity( return result - def get_entity_semityped(self, entity_urn: str) -> AspectBag: - """Get all non-timeseries aspects for an entity (experimental). + def get_entity_semityped( + self, entity_urn: str, aspects: Optional[List[str]] = None + ) -> AspectBag: + """Get (all) non-timeseries aspects for an entity. This method is called "semityped" because it returns aspects as a dictionary of properly typed objects. While the returned dictionary is constrained using a TypedDict, @@ -538,11 +540,12 @@ def get_entity_semityped(self, entity_urn: str) -> AspectBag: something, even if the entity doesn't actually exist in DataHub. :param entity_urn: The urn of the entity + :param aspects: Optional list of aspect names being requested (e.g. ["schemaMetadata", "datasetProperties"]) :returns: A dictionary of aspect name to aspect value. If an aspect is not found, it will not be present in the dictionary. The entity's key aspect will always be present. """ - response_json = self.get_entity_raw(entity_urn) + response_json = self.get_entity_raw(entity_urn, aspects) # Now, we parse the response into proper aspect objects. result: AspectBag = {} diff --git a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py index 2245e27ecedabf..33bfb63feb3fd7 100644 --- a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +++ b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py @@ -43,6 +43,7 @@ class DatahubIngestionRunSummaryProvider(PipelineRunListener): _EXECUTOR_ID: str = "__datahub_cli_" _EXECUTION_REQUEST_SOURCE_TYPE: str = "CLI_INGESTION_SOURCE" _INGESTION_TASK_NAME: str = "CLI Ingestion" + _MAX_SUMMARY_SIZE: int = 800000 @staticmethod def get_cur_time_in_ms() -> int: @@ -209,7 +210,9 @@ def on_completion( status=status, startTimeMs=self.start_time_ms, durationMs=self.get_cur_time_in_ms() - self.start_time_ms, - report=summary, + # Truncate summary such that the generated MCP will not exceed GMS's payload limit. + # Hardcoding the overall size of dataHubExecutionRequestResult to >1MB by trimming summary to 800,000 chars + report=summary[-self._MAX_SUMMARY_SIZE :], structuredReport=structured_report, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py index c9833f69825998..39ebd79c2e2269 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py @@ -198,11 +198,6 @@ class TableData: @support_status(SupportStatus.INCUBATING) @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") @capability(SourceCapability.TAGS, "Can extract ABS object/container tags if enabled") -@capability( - SourceCapability.DELETION_DETECTION, - "Optionally enabled via `stateful_ingestion.remove_stale_metadata`", - supported=True, -) class ABSSource(StatefulIngestionSourceBase): source_config: DataLakeSourceConfig report: DataLakeSourceReport diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py index feee89ba579837..e3f9a150ad0001 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py +++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py @@ -12,11 +12,12 @@ from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SupportStatus, + capability, config_class, platform_name, support_status, ) -from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.source import Source, SourceCapability, SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source_config.csv_enricher import CSVEnricherConfig from datahub.metadata.schema_classes import ( @@ -96,6 +97,10 @@ class CSVEnricherReport(SourceReport): @platform_name("CSV Enricher") @config_class(CSVEnricherConfig) @support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.DOMAINS, "Supported by default") +@capability(SourceCapability.TAGS, "Supported by default") +@capability(SourceCapability.DESCRIPTIONS, "Supported by default") +@capability(SourceCapability.OWNERSHIP, "Supported by default") class CSVEnricherSource(Source): """ :::tip Looking to ingest a CSV data file into DataHub, as an asset? diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py index 8a99f096b51676..0672b9ce6f781c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py @@ -40,8 +40,7 @@ class DBTCloudConfig(DBTCommonConfig): metadata_endpoint: str = Field( default="https://metadata.cloud.getdbt.com/graphql", - description="The dbt Cloud metadata API endpoint. This is deprecated, and will be removed in a future release. Please use access_url instead.", - deprecated=True, + description="The dbt Cloud metadata API endpoint. If not provided, we will try to infer it from the access_url.", ) token: str = Field( @@ -66,13 +65,39 @@ class DBTCloudConfig(DBTCommonConfig): @root_validator(pre=True) def set_metadata_endpoint(cls, values: dict) -> dict: if values.get("access_url") and not values.get("metadata_endpoint"): - parsed_uri = urlparse(values["access_url"]) - values[ - "metadata_endpoint" - ] = f"{parsed_uri.scheme}://metadata.{parsed_uri.netloc}/graphql" + metadata_endpoint = infer_metadata_endpoint(values["access_url"]) + if metadata_endpoint is None: + raise ValueError( + "Unable to infer the metadata endpoint from the access URL. Please provide a metadata endpoint." + ) + values["metadata_endpoint"] = metadata_endpoint return values +def infer_metadata_endpoint(access_url: str) -> Optional[str]: + # See https://docs.getdbt.com/docs/cloud/about-cloud/access-regions-ip-addresses#api-access-urls + # and https://docs.getdbt.com/docs/dbt-cloud-apis/discovery-querying#discovery-api-endpoints + + try: + parsed_uri = urlparse(access_url) + assert parsed_uri.scheme is not None + assert parsed_uri.hostname is not None + except Exception as e: + logger.debug(f"Unable to parse access URL {access_url}: {e}", exc_info=e) + return None + + if parsed_uri.hostname.endswith(".dbt.com"): + # For cell-based deployments. + # prefix.region.dbt.com -> prefix.metadata.region.dbt.com + hostname_parts = parsed_uri.hostname.split(".", maxsplit=1) + return f"{parsed_uri.scheme}://{hostname_parts[0]}.metadata.{hostname_parts[1]}/graphql" + elif parsed_uri.hostname.endswith(".getdbt.com"): + return f"{parsed_uri.scheme}://metadata.{parsed_uri.netloc}/graphql" + else: + # The self-hosted variants also have the metadata. prefix. + return f"{parsed_uri.scheme}://metadata.{parsed_uri.netloc}/graphql" + + _DBT_GRAPHQL_COMMON_FIELDS = """ runId accountId diff --git a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py index 6cab0ffc8f25c5..acda656526ef53 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py @@ -166,11 +166,6 @@ def report_dropped(self, name: str) -> None: SourceCapability.PLATFORM_INSTANCE, "By default, platform_instance will use the AWS account id", ) -@capability( - SourceCapability.DELETION_DETECTION, - "Optionally enabled via `stateful_ingestion.remove_stale_metadata`", - supported=True, -) class DynamoDBSource(StatefulIngestionSourceBase): """ This plugin extracts the following: diff --git a/metadata-ingestion/src/datahub/ingestion/source/feast.py b/metadata-ingestion/src/datahub/ingestion/source/feast.py index db0c8e9c39e7bf..e097fd1f221ea5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/feast.py +++ b/metadata-ingestion/src/datahub/ingestion/source/feast.py @@ -96,6 +96,8 @@ class FeastRepositorySourceConfig(ConfigModel): @platform_name("Feast") @config_class(FeastRepositorySourceConfig) @support_status(SupportStatus.CERTIFIED) +@capability(SourceCapability.DESCRIPTIONS, "Enabled by default") +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default") @dataclass class FeastRepositorySource(Source): diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py index 17047457e0eba0..266f9f6db57620 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py @@ -1125,6 +1125,7 @@ def transform_connector_config( @config_class(KafkaConnectSourceConfig) @support_status(SupportStatus.CERTIFIED) @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default") class KafkaConnectSource(StatefulIngestionSourceBase): config: KafkaConnectSourceConfig diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py index 35231d273fbba4..7d4ebf00cc06ef 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py @@ -1,5 +1,5 @@ from functools import lru_cache -from typing import ClassVar, Optional, TextIO, cast +from typing import ClassVar, Optional, TextIO from liquid import Environment from liquid.ast import Node @@ -25,18 +25,9 @@ def __init__(self, tok: Token, sql_or_lookml_reference: str, filter_name: str): self.filter_name = filter_name def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]: - filter_value: Optional[str] = cast( - str, context.globals.get(self.filter_name) - ) # to silent lint - - if filter_value is None: - raise CustomTagException( - f'filter {self.filter_name} value is not provided for "condition" tag' - ) - - filter_value = filter_value.strip() - - buffer.write(f"{self.sql_or_lookml_reference}='{filter_value}'") + # This implementation will make sure that sql parse work correctly if looker condition tag + # is used in lookml sql field + buffer.write(f"{self.sql_or_lookml_reference}='dummy_value'") return True diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index 919d9232a18c5c..2c523fcd98d08c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -9,6 +9,7 @@ CustomTagException, create_template, ) +from datahub.ingestion.source.looker.lookml_config import DERIVED_VIEW_PATTERN from datahub.ingestion.source.looker.str_functions import ( remove_extra_spaces_and_newlines, ) @@ -94,6 +95,24 @@ def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str: return text +def _complete_incomplete_sql(raw_view: dict, sql: str) -> str: + + # Looker supports sql fragments that omit the SELECT and FROM parts of the query + # Add those in if we detect that it is missing + sql_query: str = sql + + if not re.search(r"SELECT\s", sql_query, flags=re.I): + # add a SELECT clause at the beginning + sql_query = f"SELECT {sql}" + + if not re.search(r"FROM\s", sql_query, flags=re.I): + # add a FROM clause at the end + sql_query = f"{sql_query} FROM {raw_view['name']}" + + # Drop ${ and } + return re.sub(DERIVED_VIEW_PATTERN, r"\1", sql_query) + + def resolve_liquid_variable_in_view_dict( raw_view: dict, liquid_variable: Dict[Any, Any] ) -> None: @@ -102,14 +121,18 @@ def resolve_liquid_variable_in_view_dict( for view in raw_view["views"]: if "sql_table_name" in view: - view["sql_table_name"] = resolve_liquid_variable( + view["datahub_transformed_sql_table_name"] = resolve_liquid_variable( text=remove_extra_spaces_and_newlines(view["sql_table_name"]), liquid_variable=liquid_variable, - ) + ) # keeping original sql_table_name as is to avoid any visualization issue later if "derived_table" in view and "sql" in view["derived_table"]: # In sql we don't need to remove the extra spaces as sql parser takes care of extra spaces and \n # while generating URN from sql - view["derived_table"]["sql"] = resolve_liquid_variable( + view["derived_table"]["datahub_transformed_sql"] = resolve_liquid_variable( text=view["derived_table"]["sql"], liquid_variable=liquid_variable + ) # keeping original sql as is, so that on UI sql will be shown same is it is visible on looker portal + + view["derived_table"]["datahub_transformed_sql"] = _complete_incomplete_sql( + raw_view=view, sql=view["derived_table"]["datahub_transformed_sql"] ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py index e528e578dcf9fa..a83aa2638ec964 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py @@ -266,15 +266,25 @@ def sql_table_name(self) -> str: sql_table_name: Optional[str] = self._get_sql_table_name_field() # if sql_table_name field is not set then the table name is equal to view-name if sql_table_name is None: - return self.raw_view[NAME].lower() + sql_table_name = self.raw_view[NAME].lower() + + return sql_table_name + + def datahub_transformed_sql_table_name(self) -> str: + table_name: Optional[str] = self.raw_view.get( + "datahub_transformed_sql_table_name" + ) + + if not table_name: + table_name = self.sql_table_name() # sql_table_name is in the format "${view-name}.SQL_TABLE_NAME" # remove extra characters if self._is_dot_sql_table_name_present(): - sql_table_name = re.sub(DERIVED_VIEW_PATTERN, r"\1", sql_table_name) + table_name = re.sub(DERIVED_VIEW_PATTERN, r"\1", table_name) # Some sql_table_name fields contain quotes like: optimizely."group", just remove the quotes - return sql_table_name.replace('"', "").replace("`", "").lower() + return table_name.replace('"', "").replace("`", "").lower() def derived_table(self) -> Dict[Any, Any]: """ @@ -296,30 +306,21 @@ def explore_source(self) -> Dict[Any, Any]: return derived_table["explore_source"] - def sql(self, transformed: bool = True) -> str: + def sql(self) -> str: """ This function should only be called if is_sql_based_derived_case return true """ derived_table = self.derived_table() - # Looker supports sql fragments that omit the SELECT and FROM parts of the query - # Add those in if we detect that it is missing - sql_query: str = derived_table["sql"] - - if transformed: # update the original sql attribute only if transformed is true - if not re.search(r"SELECT\s", sql_query, flags=re.I): - # add a SELECT clause at the beginning - sql_query = f"SELECT {sql_query}" + return derived_table["sql"] - if not re.search(r"FROM\s", sql_query, flags=re.I): - # add a FROM clause at the end - sql_query = f"{sql_query} FROM {self.name()}" - # Get the list of tables in the query - - # Drop ${ and } - sql_query = re.sub(DERIVED_VIEW_PATTERN, r"\1", sql_query) + def datahub_transformed_sql(self) -> str: + """ + This function should only be called if is_sql_based_derived_case return true + """ + derived_table = self.derived_table() - return sql_query + return derived_table["datahub_transformed_sql"] def name(self) -> str: return self.raw_view[NAME] diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index aa5719547c03ed..f4fb1316b16a20 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -11,8 +11,10 @@ from datahub.configuration.git import GitInfo from datahub.configuration.source_common import EnvConfigMixin from datahub.configuration.validate_field_rename import pydantic_renamed_field -from datahub.ingestion.source.looker.looker_config import LookerCommonConfig -from datahub.ingestion.source.looker.looker_connection import LookerConnectionDefinition +from datahub.ingestion.source.looker.looker_config import ( + LookerCommonConfig, + LookerConnectionDefinition, +) from datahub.ingestion.source.looker.looker_lib_wrapper import ( LookerAPI, LookerAPIConfig, diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 223d168dbe033a..d77e65ac733232 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -201,8 +201,7 @@ def from_looker_dict( view_logic = view_context.view_file.raw_file_content[:max_file_snippet_length] if view_context.is_sql_based_derived_case(): - view_logic = view_context.sql(transformed=False) - # Parse SQL to extract dependencies. + view_logic = view_context.sql() view_details = ViewProperties( materialized=False, viewLogic=view_logic, diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py index 390e71ef9d4bd8..98646e19a7014b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py @@ -206,6 +206,7 @@ class AbstractViewUpstream(ABC): view_context: LookerViewContext looker_view_id_cache: LookerViewIdCache config: LookMLSourceConfig + reporter: LookMLSourceReport ctx: PipelineContext def __init__( @@ -213,11 +214,13 @@ def __init__( view_context: LookerViewContext, looker_view_id_cache: LookerViewIdCache, config: LookMLSourceConfig, + reporter: LookMLSourceReport, ctx: PipelineContext, ): self.view_context = view_context self.looker_view_id_cache = looker_view_id_cache self.config = config + self.reporter = reporter self.ctx = ctx @abstractmethod @@ -244,9 +247,10 @@ def __init__( view_context: LookerViewContext, looker_view_id_cache: LookerViewIdCache, config: LookMLSourceConfig, + reporter: LookMLSourceReport, ctx: PipelineContext, ): - super().__init__(view_context, looker_view_id_cache, config, ctx) + super().__init__(view_context, looker_view_id_cache, config, reporter, ctx) # These are the function where we need to catch the response once calculated self._get_spr = lru_cache(maxsize=1)(self.__get_spr) self._get_upstream_dataset_urn = lru_cache(maxsize=1)( @@ -259,7 +263,7 @@ def __get_spr(self) -> Optional[SqlParsingResult]: return None spr = create_lineage_sql_parsed_result( - query=self.view_context.sql(), + query=self.view_context.datahub_transformed_sql(), default_schema=self.view_context.view_connection.default_schema, default_db=self.view_context.view_connection.default_db, platform=self.view_context.view_connection.platform, @@ -267,17 +271,6 @@ def __get_spr(self) -> Optional[SqlParsingResult]: env=self.view_context.view_connection.platform_env or self.config.env, graph=self.ctx.graph, ) - - if ( - spr.debug_info.table_error is not None - or spr.debug_info.column_error is not None - ): - logging.debug( - f"Failed to parsed the sql query. table_error={spr.debug_info.table_error} and " - f"column_error={spr.debug_info.column_error}" - ) - return None - return spr def __get_upstream_dataset_urn(self) -> List[Urn]: @@ -286,6 +279,15 @@ def __get_upstream_dataset_urn(self) -> List[Urn]: if sql_parsing_result is None: return [] + if sql_parsing_result.debug_info.table_error is not None: + self.reporter.report_warning( + title="Table Level Lineage Missing", + message="Error in parsing derived sql", + context=f"View-name: {self.view_context.name()}", + exc=sql_parsing_result.debug_info.table_error, + ) + return [] + upstream_dataset_urns: List[str] = [ _drop_hive_dot(urn) for urn in sql_parsing_result.in_tables ] @@ -306,6 +308,15 @@ def create_fields(self) -> List[ViewField]: if spr is None: return [] + if spr.debug_info.column_error is not None: + self.reporter.report_warning( + title="Column Level Lineage Missing", + message="Error in parsing derived sql for CLL", + context=f"View-name: {self.view_context.name()}", + exc=spr.debug_info.column_error, + ) + return [] + fields: List[ViewField] = [] column_lineages: List[ColumnLineageInfo] = ( @@ -336,6 +347,15 @@ def get_upstream_column_ref( if sql_parsing_result is None: return [] + if sql_parsing_result.debug_info.column_error is not None: + self.reporter.report_warning( + title="Column Level Lineage Missing", + message="Error in parsing derived sql for CLL", + context=f"View-name: {self.view_context.name()}. " + f"Error: {sql_parsing_result.debug_info.column_error}", + ) + return [] + upstreams_column_refs: List[ColumnRef] = [] if sql_parsing_result.column_lineage: for cll in sql_parsing_result.column_lineage: @@ -384,9 +404,11 @@ def __init__( view_context: LookerViewContext, looker_view_id_cache: LookerViewIdCache, config: LookMLSourceConfig, + reporter: LookMLSourceReport, ctx: PipelineContext, ): - super().__init__(view_context, looker_view_id_cache, config, ctx) + super().__init__(view_context, looker_view_id_cache, config, reporter, ctx) + self._get_upstream_dataset_urn = lru_cache(maxsize=1)( self.__get_upstream_dataset_urn ) @@ -402,7 +424,7 @@ def __get_upstream_dataset_urn(self) -> List[str]: base_folder_path=self.view_context.base_folder_path, ) - # Current view will always be present in cache. The assert will silence the lint + # Current view will always be present in cache. assert will silence the lint assert current_view_id # We're creating a "LookerExplore" just to use the urn generator. @@ -467,9 +489,10 @@ def __init__( view_context: LookerViewContext, looker_view_id_cache: LookerViewIdCache, config: LookMLSourceConfig, + reporter: LookMLSourceReport, ctx: PipelineContext, ): - super().__init__(view_context, looker_view_id_cache, config, ctx) + super().__init__(view_context, looker_view_id_cache, config, reporter, ctx) self.upstream_dataset_urn = None self._get_upstream_dataset_urn = lru_cache(maxsize=1)( @@ -478,9 +501,9 @@ def __init__( def __get_upstream_dataset_urn(self) -> Urn: # In regular case view's upstream dataset is either same as view-name or mentioned in "sql_table_name" field - # view_context.sql_table_name() handle this condition to return dataset name + # view_context.datahub_transformed_sql_table_name() handle this condition to return dataset name qualified_table_name: str = _generate_fully_qualified_name( - sql_table_name=self.view_context.sql_table_name(), + sql_table_name=self.view_context.datahub_transformed_sql_table_name(), connection_def=self.view_context.view_connection, reporter=self.view_context.reporter, ) @@ -522,9 +545,10 @@ def __init__( view_context: LookerViewContext, looker_view_id_cache: LookerViewIdCache, config: LookMLSourceConfig, + reporter: LookMLSourceReport, ctx: PipelineContext, ): - super().__init__(view_context, looker_view_id_cache, config, ctx) + super().__init__(view_context, looker_view_id_cache, config, reporter, ctx) self.upstream_dataset_urn = [] self._get_upstream_dataset_urn = lru_cache(maxsize=1)( @@ -532,10 +556,10 @@ def __init__( ) def __get_upstream_dataset_urn(self) -> List[Urn]: - # In this case view_context.sql_table_name() refers to derived view name + # In this case view_context.datahub_transformed_sql_table_name() refers to derived view name looker_view_id = get_derived_looker_view_id( qualified_table_name=_generate_fully_qualified_name( - self.view_context.sql_table_name(), + self.view_context.datahub_transformed_sql_table_name(), self.view_context.view_connection, self.view_context.reporter, ), @@ -591,6 +615,7 @@ def create_view_upstream( return RegularViewUpstream( view_context=view_context, config=config, + reporter=reporter, ctx=ctx, looker_view_id_cache=looker_view_id_cache, ) @@ -599,6 +624,7 @@ def create_view_upstream( return DotSqlTableNameViewUpstream( view_context=view_context, config=config, + reporter=reporter, ctx=ctx, looker_view_id_cache=looker_view_id_cache, ) @@ -610,6 +636,7 @@ def create_view_upstream( return SqlBasedDerivedViewUpstream( view_context=view_context, config=config, + reporter=reporter, ctx=ctx, looker_view_id_cache=looker_view_id_cache, ) @@ -618,6 +645,7 @@ def create_view_upstream( return NativeDerivedViewUpstream( view_context=view_context, config=config, + reporter=reporter, ctx=ctx, looker_view_id_cache=looker_view_id_cache, ) @@ -631,6 +659,7 @@ def create_view_upstream( return EmptyImplementation( view_context=view_context, config=config, + reporter=reporter, ctx=ctx, looker_view_id_cache=looker_view_id_cache, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py index 4b4822bcb98cae..47475c5825a493 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mode.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py @@ -135,9 +135,14 @@ class ModeConfig(StatefulIngestionConfigBase, DatasetLineageProviderConfigBase): connect_uri: str = Field( default="https://app.mode.com", description="Mode host URL." ) - token: str = Field(description="Mode user token.") + token: str = Field( + description="When creating workspace API key this is the 'Key ID'." + ) password: pydantic.SecretStr = Field( - description="Mode password for authentication." + description="When creating workspace API key this is the 'Secret'." + ) + exclude_restricted: bool = Field( + default=False, description="Exclude restricted collections" ) workspace: str = Field( @@ -205,8 +210,12 @@ def report_dropped_space(self, ent_name: str) -> None: @platform_name("Mode") @config_class(ModeConfig) @support_status(SupportStatus.CERTIFIED) +@capability(SourceCapability.CONTAINERS, "Enabled by default") +@capability(SourceCapability.DESCRIPTIONS, "Enabled by default") @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") @capability(SourceCapability.LINEAGE_COARSE, "Supported by default") +@capability(SourceCapability.LINEAGE_FINE, "Supported by default") +@capability(SourceCapability.OWNERSHIP, "Enabled by default") class ModeSource(StatefulIngestionSourceBase): """ @@ -522,6 +531,16 @@ def _get_space_name_and_tokens(self) -> dict: for s in spaces: logger.debug(f"Space: {s.get('name')}") space_name = s.get("name", "") + # Using both restricted and default_access_level because + # there is a current bug with restricted returning False everytime + # which has been reported to Mode team + if self.config.exclude_restricted and ( + s.get("restricted") or s.get("default_access_level") == "restricted" + ): + logging.debug( + f"Skipping space {space_name} due to exclude restricted" + ) + continue if not self.config.space_pattern.allowed(space_name): self.report.report_dropped_space(space_name) logging.debug(f"Skipping space {space_name} due to space pattern") @@ -741,7 +760,7 @@ def _replace_definitions(self, raw_query: str) -> str: def _parse_definition_name(self, definition_variable: str) -> Tuple[str, str]: name, alias = "", "" # i.e '{{ @join_on_definition as alias}}' - name_match = re.findall("@[a-zA-z]+", definition_variable) + name_match = re.findall("@[a-zA-Z_]+", definition_variable) if len(name_match): name = name_match[0][1:] alias_match = re.findall( diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py index bd80433bc2e6ca..967dd5d81112d5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py @@ -440,6 +440,11 @@ class PowerBiDashboardSourceConfig( ) profiling: PowerBiProfilingConfig = PowerBiProfilingConfig() + patch_metadata: bool = pydantic.Field( + default=True, + description="Patch dashboard metadata", + ) + @root_validator(skip_on_failure=True) def validate_extract_column_level_lineage(cls, values: Dict) -> Dict: flags = [ diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index de4eaf6b64434f..e0a72c71a1ef00 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -1197,8 +1197,7 @@ def report_to_datahub_work_units( ) -> Iterable[MetadataWorkUnit]: mcps: List[MetadataChangeProposalWrapper] = [] - logger.debug(f"Converting dashboard={report.name} to datahub dashboard") - + logger.debug(f"Converting report={report.name} to datahub dashboard") # Convert user to CorpUser user_mcps = self.to_datahub_users(report.users) # Convert pages to charts. A report has single dataset and same dataset used in pages to create visualization @@ -1215,16 +1214,18 @@ def report_to_datahub_work_units( mcps.extend(chart_mcps) mcps.extend(report_mcps) - # Convert MCP to work_units - work_units = map(self._to_work_unit, mcps) - return work_units + return map(self._to_work_unit, mcps) @platform_name("PowerBI") @config_class(PowerBiDashboardSourceConfig) @support_status(SupportStatus.CERTIFIED) +@capability(SourceCapability.CONTAINERS, "Enabled by default") @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") +@capability(SourceCapability.OWNERSHIP, "Enabled by default") @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") +@capability(SourceCapability.TAGS, "Enabled by default") @capability( SourceCapability.OWNERSHIP, "Disabled by default, configured using `extract_ownership`", @@ -1385,7 +1386,7 @@ def _get_dashboard_patch_work_unit( DashboardInfoClass ] = work_unit.get_aspect_of_type(DashboardInfoClass) - if dashboard_info_aspect: + if dashboard_info_aspect and self.source_config.patch_metadata: return convert_dashboard_info_to_patch( work_unit.get_urn(), dashboard_info_aspect, diff --git a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py index 7671e239284305..790c1f918cdfd2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py +++ b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py @@ -91,6 +91,7 @@ def __init__(self, schema): @config_class(PulsarSourceConfig) @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field") +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") @dataclass class PulsarSource(StatefulIngestionSourceBase): def __init__(self, config: PulsarSourceConfig, ctx: PipelineContext): diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index b9fd2a9c4fe221..b6c48dd3c488ec 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -93,12 +93,22 @@ @platform_name("Qlik Sense") @config_class(QlikSourceConfig) @support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.CONTAINERS, "Enabled by default") @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") +@capability( + SourceCapability.LINEAGE_COARSE, + "Enabled by default.", +) +@capability( + SourceCapability.LINEAGE_FINE, + "Disabled by default. ", +) @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") @capability( SourceCapability.OWNERSHIP, "Enabled by default, configured using `ingest_owner`", ) +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") class QlikSenseSource(StatefulIngestionSourceBase, TestableSource): """ This plugin extracts the following: diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py index c7a3f25e947dc5..38cf0bebcbc12f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redash.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redash.py @@ -18,12 +18,13 @@ from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( # SourceCapability,; capability, SupportStatus, + capability, config_class, platform_name, support_status, ) from datahub.ingestion.api.registry import import_path -from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.source import Source, SourceCapability, SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.common import ( AuditStamp, @@ -308,6 +309,7 @@ def report_dropped(self, item: str) -> None: @platform_name("Redash") @config_class(RedashConfig) @support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default") class RedashSource(Source): """ This plugin extracts the following: diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py index a6ffed65aaa70c..a9fc9ab8f3e993 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py @@ -120,7 +120,7 @@ @platform_name("Redshift") @config_class(RedshiftConfig) @support_status(SupportStatus.CERTIFIED) -@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability(SourceCapability.CONTAINERS, "Enabled by default") @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field") @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") @@ -129,6 +129,8 @@ SourceCapability.LINEAGE_FINE, "Optionally enabled via configuration (`mixed` or `sql_based` lineage needs to be enabled)", ) +@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") @capability( SourceCapability.USAGE_STATS, "Enabled by default, can be disabled via configuration `include_usage_statistics`", diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py index c35f500df1b8c7..b8c7fd5aa88fc1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py @@ -220,13 +220,12 @@ class TableData: @platform_name("S3 / Local Files", id="s3") @config_class(DataLakeSourceConfig) @support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.CONTAINERS, "Enabled by default") @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") -@capability(SourceCapability.TAGS, "Can extract S3 object/bucket tags if enabled") @capability( - SourceCapability.DELETION_DETECTION, - "Optionally enabled via `stateful_ingestion.remove_stale_metadata`", - supported=True, + SourceCapability.SCHEMA_METADATA, "Can infer schema from supported file types" ) +@capability(SourceCapability.TAGS, "Can extract S3 object/bucket tags if enabled") class S3Source(StatefulIngestionSourceBase): source_config: DataLakeSourceConfig report: DataLakeSourceReport diff --git a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py index 946fdcedc571f8..42128123c61442 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py +++ b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py @@ -199,6 +199,14 @@ def report_dropped(self, ent_name: str) -> None: description="Not supported yet", supported=False, ) +@capability( + capability_name=SourceCapability.SCHEMA_METADATA, + description="Enabled by default", +) +@capability( + capability_name=SourceCapability.TAGS, + description="Enabled by default", +) class SalesforceSource(Source): base_url: str config: SalesforceConfig diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py index 74d7abb121a3eb..5db5e543510db9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -86,8 +86,12 @@ @platform_name("Sigma") @config_class(SigmaSourceConfig) @support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.CONTAINERS, "Enabled by default") @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") +@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default.") @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") +@capability(SourceCapability.TAGS, "Enabled by default") @capability( SourceCapability.OWNERSHIP, "Enabled by default, configured using `ingest_owner`", diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index 398adc3708ef2b..5cc51882965a0c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -499,6 +499,7 @@ def get_schema_fields_for_column( self, dataset_name: str, column: Dict, + inspector: Inspector, pk_constraints: Optional[dict] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, @@ -506,6 +507,7 @@ def get_schema_fields_for_column( fields = get_schema_fields_for_sqlalchemy_column( column_name=column["name"], column_type=column["type"], + inspector=inspector, description=column.get("comment", None), nullable=column.get("nullable", True), is_part_of_key=( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py index 65f8516fd340a3..59f301baf40165 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py @@ -169,12 +169,16 @@ def get_schema_fields_for_column( self, dataset_name: str, column: Dict[Any, Any], + inspector: Inspector, pk_constraints: Optional[Dict[Any, Any]] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, ) -> List[SchemaField]: fields = super().get_schema_fields_for_column( - dataset_name, column, pk_constraints + dataset_name, + column, + inspector, + pk_constraints, ) if self._COMPLEX_TYPE.match(fields[0].nativeDataType) and isinstance( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py index 655d1ba68ed79e..9da6c294881247 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py @@ -521,7 +521,7 @@ def loop_tables( ) # add table schema fields - schema_fields = self.get_schema_fields(dataset_name, columns) + schema_fields = self.get_schema_fields(dataset_name, columns, inspector) self._set_partition_key(columns, schema_fields) @@ -754,7 +754,9 @@ def loop_views( # add view schema fields schema_fields = self.get_schema_fields( - dataset.dataset_name, dataset.columns + dataset.dataset_name, + dataset.columns, + inspector, ) schema_metadata = get_schema_metadata( @@ -877,6 +879,7 @@ def get_schema_fields_for_column( self, dataset_name: str, column: Dict[Any, Any], + inspector: Inspector, pk_constraints: Optional[Dict[Any, Any]] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index b3a5f134c61d64..1fa308eae6b769 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -120,6 +120,9 @@ from datahub.utilities.lossy_collections import LossyList from datahub.utilities.registries.domain_registry import DomainRegistry from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport +from datahub.utilities.sqlalchemy_type_converter import ( + get_native_data_type_for_sqlalchemy_type, +) if TYPE_CHECKING: from datahub.ingestion.source.ge_data_profiler import ( @@ -320,6 +323,26 @@ class ProfileMetadata: "Optionally enabled via `classification.enabled`", supported=True, ) +@capability( + SourceCapability.SCHEMA_METADATA, + "Enabled by default", + supported=True, +) +@capability( + SourceCapability.CONTAINERS, + "Enabled by default", + supported=True, +) +@capability( + SourceCapability.DESCRIPTIONS, + "Enabled by default", + supported=True, +) +@capability( + SourceCapability.DOMAINS, + "Enabled by default", + supported=True, +) class SQLAlchemySource(StatefulIngestionSourceBase, TestableSource): """A Base class for all SQL Sources that use SQLAlchemy to extend""" @@ -787,6 +810,7 @@ def _process_table( schema_fields = self.get_schema_fields( dataset_name, columns, + inspector, pk_constraints, tags=extra_tags, partition_keys=partitions, @@ -967,6 +991,7 @@ def get_schema_fields( self, dataset_name: str, columns: List[dict], + inspector: Inspector, pk_constraints: Optional[dict] = None, partition_keys: Optional[List[str]] = None, tags: Optional[Dict[str, List[str]]] = None, @@ -979,6 +1004,7 @@ def get_schema_fields( fields = self.get_schema_fields_for_column( dataset_name, column, + inspector, pk_constraints, tags=column_tags, partition_keys=partition_keys, @@ -990,6 +1016,7 @@ def get_schema_fields_for_column( self, dataset_name: str, column: dict, + inspector: Inspector, pk_constraints: Optional[dict] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, @@ -999,10 +1026,16 @@ def get_schema_fields_for_column( tags_str = [make_tag_urn(t) for t in tags] tags_tac = [TagAssociationClass(t) for t in tags_str] gtc = GlobalTagsClass(tags_tac) + full_type = column.get("full_type") field = SchemaField( fieldPath=column["name"], type=get_column_type(self.report, dataset_name, column["type"]), - nativeDataType=column.get("full_type", repr(column["type"])), + nativeDataType=full_type + if full_type is not None + else get_native_data_type_for_sqlalchemy_type( + column["type"], + inspector=inspector, + ), description=column.get("comment", None), nullable=column["nullable"], recursive=False, @@ -1079,7 +1112,7 @@ def _process_view( self.warn(logger, dataset_name, "unable to get schema for this view") schema_metadata = None else: - schema_fields = self.get_schema_fields(dataset_name, columns) + schema_fields = self.get_schema_fields(dataset_name, columns, inspector) schema_metadata = get_schema_metadata( self.report, dataset_name, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py index cc0a43bc5e8749..b6fa51dd70e18d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py @@ -387,12 +387,16 @@ def get_schema_fields_for_column( self, dataset_name: str, column: dict, + inspector: Inspector, pk_constraints: Optional[dict] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, ) -> List[SchemaField]: fields = super().get_schema_fields_for_column( - dataset_name, column, pk_constraints + dataset_name, + column, + inspector, + pk_constraints, ) if isinstance(column["type"], (datatype.ROW, sqltypes.ARRAY, datatype.MAP)): diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index ae56fb87ee5281..a340f049731c46 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -469,7 +469,12 @@ def _process_projections( foreign_keys = self._get_foreign_keys( dataset_urn, inspector, schema, projection ) - schema_fields = self.get_schema_fields(dataset_name, columns, pk_constraints) + schema_fields = self.get_schema_fields( + dataset_name, + columns, + inspector, + pk_constraints, + ) schema_metadata = get_schema_metadata( self.report, dataset_name, @@ -673,7 +678,7 @@ def _process_models( ) dataset_snapshot.aspects.append(dataset_properties) - schema_fields = self.get_schema_fields(dataset_name, columns) + schema_fields = self.get_schema_fields(dataset_name, columns, inspector) schema_metadata = get_schema_metadata( self.report, diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py index 0f031177c403a3..f011aa7bdd19e4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py @@ -131,6 +131,9 @@ def get_percent_entities_changed( new_entities=self.urns, old_entities=old_checkpoint_state.urns ) + def urn_count(self) -> int: + return len(self.urns) + def compute_percent_entities_changed( new_entities: List[str], old_entities: List[str] diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py index ee1ccdff781dc1..c73472f1b8041e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py @@ -42,7 +42,7 @@ class StatefulStaleMetadataRemovalConfig(StatefulIngestionConfig): description="Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", ) fail_safe_threshold: float = pydantic.Field( - default=40.0, + default=75.0, description="Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", le=100.0, ge=0.0, @@ -257,13 +257,33 @@ def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: assert self.stateful_ingestion_config - copy_previous_state_and_fail = False + copy_previous_state_and_exit = False + + # If the source already had a failure, skip soft-deletion. + # TODO: Eventually, switch this to check if anything in the pipeline had a failure so far, not just the source. + if self.source.get_report().failures: + self.source.get_report().report_warning( + title="Skipping stateful ingestion / stale entity removal", + message="The soft-deletion of stale entities will be skipped because the source reported a failure.", + ) + copy_previous_state_and_exit = True + + if ( + not copy_previous_state_and_exit + and self.source.get_report().events_produced == 0 + ): + self.source.get_report().report_failure( + title="Skipping stateful ingestion / stale entity removal", + message="The source did not produce any metadata. Despite stateful ingestion being enabled, we will not delete any metadata. " + "This is a fail-safe mechanism to prevent the accidental deletion of all entities.", + ) + copy_previous_state_and_exit = True # Check if the entity delta is below the fail-safe threshold. entity_difference_percent = cur_checkpoint_state.get_percent_entities_changed( last_checkpoint_state ) - if ( + if not copy_previous_state_and_exit and ( entity_difference_percent > self.stateful_ingestion_config.fail_safe_threshold # Adding this check to protect against cases where get_percent_entities_changed returns over 100%. @@ -273,32 +293,23 @@ def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: ): # Log the failure. This would prevent the current state from getting committed. self.source.get_report().report_failure( - "stale-entity-removal", - f"Will not soft-delete entities, since we'd be deleting {entity_difference_percent:.1f}% of the existing entities. " - f"To force a deletion, increase the value of 'stateful_ingestion.fail_safe_threshold' (currently {self.stateful_ingestion_config.fail_safe_threshold})", - ) - copy_previous_state_and_fail = True - - if self.source.get_report().events_produced == 0: - self.source.get_report().report_failure( - "stale-entity-removal", - "Skipping stale entity soft-deletion because the source produced no events. " - "This is a fail-safe mechanism to prevent accidental deletion of all entities.", - ) - copy_previous_state_and_fail = True - - # If the source already had a failure, skip soft-deletion. - # TODO: Eventually, switch this to check if anything in the pipeline had a failure so far, not just the source. - if self.source.get_report().failures: - self.source.get_report().report_warning( - "stale-entity-removal", - "Skipping stale entity soft-deletion and copying urns from last state since source already had failures.", + title="Skipping stateful ingestion / stale entity removal", + message=f"\ +The previous run produced {last_checkpoint_state.urn_count()} entities, whereas this run produced {cur_checkpoint_state.urn_count()} entities. \ +Comparing the entities produced this run vs the previous run, we would be deleting {entity_difference_percent:.1f}% of the entities produced by the previous run. \ +This percentage is above the threshold (currently {self.stateful_ingestion_config.fail_safe_threshold}), so we will skip soft-deleting stale entities.\ +\ +To update this threshold, add this to your recipe: \ +\ +stateful_ingestion:\ + fail_safe_threshold: \ +", ) - copy_previous_state_and_fail = True + copy_previous_state_and_exit = True - if copy_previous_state_and_fail: + if copy_previous_state_and_exit: logger.info( - f"Copying urns from last state (size {last_checkpoint_state.urns}) to current state (size {cur_checkpoint_state.urns}) " + f"Copying urns from last state (size {len(last_checkpoint_state.urns)}) to current state (size {len(cur_checkpoint_state.urns)}) " "to ensure stale entities from previous runs are deleted on the next successful run." ) for urn in last_checkpoint_state.urns: diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 5141b8b080bca8..5af5a5a6ca0244 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -162,6 +162,7 @@ def get_filter_name(filter_obj): @capability( SourceCapability.DELETION_DETECTION, "Optionally enabled via stateful_ingestion" ) +@capability(SourceCapability.DOMAINS, "Enabled by `domain` config to assign domain_key") @capability(SourceCapability.LINEAGE_COARSE, "Supported by default") class SupersetSource(StatefulIngestionSourceBase): """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index 1655724f2d402d..9cde3b1f8d3a07 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -1,5 +1,6 @@ import logging import re +import time from collections import OrderedDict from dataclasses import dataclass from datetime import datetime @@ -13,6 +14,7 @@ Optional, Set, Tuple, + Type, Union, cast, ) @@ -158,6 +160,21 @@ from datahub.utilities import config_clean from datahub.utilities.urns.dataset_urn import DatasetUrn +try: + # On earlier versions of the tableauserverclient, the NonXMLResponseError + # was thrown when reauthentication was needed. We'll keep both exceptions + # around for now, but can remove this in the future. + from tableauserverclient.server.endpoint.exceptions import ( # type: ignore + NotSignedInError, + ) + + REAUTHENTICATE_ERRORS: Tuple[Type[Exception], ...] = ( + NotSignedInError, + NonXMLResponseError, + ) +except ImportError: + REAUTHENTICATE_ERRORS = (NonXMLResponseError,) + logger: logging.Logger = logging.getLogger(__name__) # Replace / with | @@ -965,7 +982,7 @@ def get_connection_object_page( query_data = query_metadata( self.server, query, connection_type, count, offset, query_filter ) - except NonXMLResponseError: + except REAUTHENTICATE_ERRORS: if not retry_on_auth_error: raise @@ -1038,6 +1055,35 @@ def get_connection_object_page( ) else: + # As of Tableau Server 2024.2, the metadata API sporadically returns a 30 second + # timeout error. It doesn't reliably happen, so retrying a couple times makes sense. + if all( + error.get("message") + == "Execution canceled because timeout of 30000 millis was reached" + for error in errors + ): + # If it was only a timeout error, we can retry. + if retries_remaining <= 0: + raise + + # This is a pretty dumb backoff mechanism, but it's good enough for now. + backoff_time = min( + (self.config.max_retries - retries_remaining + 1) ** 2, 60 + ) + logger.info( + f"Query {connection_type} received a 30 second timeout error - will retry in {backoff_time} seconds. " + f"Retries remaining: {retries_remaining}" + ) + time.sleep(backoff_time) + return self.get_connection_object_page( + query, + connection_type, + query_filter, + count, + offset, + retry_on_auth_error=False, + retries_remaining=retries_remaining - 1, + ) raise RuntimeError(f"Query {connection_type} error: {errors}") connection_object = query_data.get(c.DATA, {}).get(connection_type, {}) diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index b29170cb2d705d..9a6cde78cf10d3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -140,6 +140,9 @@ @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field") @capability(SourceCapability.CONTAINERS, "Enabled by default") @capability(SourceCapability.OWNERSHIP, "Supported via the `include_ownership` config") +@capability( + SourceCapability.DATA_PROFILING, "Supported via the `profiling.enabled` config" +) @capability( SourceCapability.DELETION_DETECTION, "Optionally enabled via `stateful_ingestion.remove_stale_metadata`", diff --git a/metadata-ingestion/src/datahub/specific/chart.py b/metadata-ingestion/src/datahub/specific/chart.py index 51994ad1d063eb..cc68168b68db7e 100644 --- a/metadata-ingestion/src/datahub/specific/chart.py +++ b/metadata-ingestion/src/datahub/specific/chart.py @@ -3,8 +3,11 @@ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( + AccessLevelClass, AuditStampClass, + ChangeAuditStampsClass, ChartInfoClass as ChartInfo, + ChartTypeClass, EdgeClass as Edge, GlobalTagsClass as GlobalTags, GlossaryTermAssociationClass as Term, @@ -311,3 +314,108 @@ def remove_custom_property(self, key: str) -> "ChartPatchBuilder": """ self.custom_properties_patch_helper.remove_property(key) return self + + def set_title(self, title: str) -> "ChartPatchBuilder": + assert title, "ChartInfo title should not be None" + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/title", + value=title, + ) + + return self + + def set_description(self, description: str) -> "ChartPatchBuilder": + assert description, "DashboardInfo description should not be None" + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/description", + value=description, + ) + + return self + + def set_last_refreshed(self, last_refreshed: Optional[int]) -> "ChartPatchBuilder": + if last_refreshed: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/lastRefreshed", + value=last_refreshed, + ) + + return self + + def set_last_modified( + self, last_modified: "ChangeAuditStampsClass" + ) -> "ChartPatchBuilder": + if last_modified: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/lastModified", + value=last_modified, + ) + + return self + + def set_external_url(self, external_url: Optional[str]) -> "ChartPatchBuilder": + if external_url: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/externalUrl", + value=external_url, + ) + return self + + def set_chart_url(self, dashboard_url: Optional[str]) -> "ChartPatchBuilder": + if dashboard_url: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/chartUrl", + value=dashboard_url, + ) + + return self + + def set_type( + self, type: Union[None, Union[str, "ChartTypeClass"]] = None + ) -> "ChartPatchBuilder": + if type: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/type", + value=type, + ) + + return self + + def set_access( + self, access: Union[None, Union[str, "AccessLevelClass"]] = None + ) -> "ChartPatchBuilder": + if access: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/access", + value=access, + ) + + return self + + def add_inputs(self, input_urns: Optional[List[str]]) -> "ChartPatchBuilder": + if input_urns: + for urn in input_urns: + self._add_patch( + aspect_name=ChartInfo.ASPECT_NAME, + op="add", + path=f"/inputs/{urn}", + value=urn, + ) + + return self diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py index e6d911b5986550..8228dbc011db2f 100644 --- a/metadata-ingestion/src/datahub/specific/dashboard.py +++ b/metadata-ingestion/src/datahub/specific/dashboard.py @@ -3,7 +3,9 @@ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( + AccessLevelClass, AuditStampClass, + ChangeAuditStampsClass, DashboardInfoClass as DashboardInfo, EdgeClass as Edge, GlobalTagsClass as GlobalTags, @@ -405,3 +407,123 @@ def remove_custom_property(self, key: str) -> "DashboardPatchBuilder": """ self.custom_properties_patch_helper.remove_property(key) return self + + def set_title(self, title: str) -> "DashboardPatchBuilder": + assert title, "DashboardInfo title should not be None" + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/title", + value=title, + ) + + return self + + def set_description(self, description: str) -> "DashboardPatchBuilder": + assert description, "DashboardInfo description should not be None" + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/description", + value=description, + ) + + return self + + def add_custom_properties( + self, custom_properties: Optional[Dict[str, str]] = None + ) -> "DashboardPatchBuilder": + + if custom_properties: + for key, value in custom_properties.items(): + self.custom_properties_patch_helper.add_property(key, value) + + return self + + def set_external_url(self, external_url: Optional[str]) -> "DashboardPatchBuilder": + if external_url: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/externalUrl", + value=external_url, + ) + return self + + def add_charts(self, chart_urns: Optional[List[str]]) -> "DashboardPatchBuilder": + if chart_urns: + for urn in chart_urns: + self._add_patch( + aspect_name=DashboardInfo.ASPECT_NAME, + op="add", + path=f"/charts/{urn}", + value=urn, + ) + + return self + + def add_datasets( + self, dataset_urns: Optional[List[str]] + ) -> "DashboardPatchBuilder": + if dataset_urns: + for urn in dataset_urns: + self._add_patch( + aspect_name=DashboardInfo.ASPECT_NAME, + op="add", + path=f"/datasets/{urn}", + value=urn, + ) + + return self + + def set_dashboard_url( + self, dashboard_url: Optional[str] + ) -> "DashboardPatchBuilder": + if dashboard_url: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/dashboardUrl", + value=dashboard_url, + ) + + return self + + def set_access( + self, access: Union[None, Union[str, "AccessLevelClass"]] = None + ) -> "DashboardPatchBuilder": + if access: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/access", + value=access, + ) + + return self + + def set_last_refreshed( + self, last_refreshed: Optional[int] + ) -> "DashboardPatchBuilder": + if last_refreshed: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/lastRefreshed", + value=last_refreshed, + ) + + return self + + def set_last_modified( + self, last_modified: "ChangeAuditStampsClass" + ) -> "DashboardPatchBuilder": + if last_modified: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/lastModified", + value=last_modified, + ) + + return self diff --git a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py index 41d02646fdb8a0..ad94c6904e2807 100644 --- a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py +++ b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py @@ -5,6 +5,8 @@ from typing import Any, Dict, List, Optional, Type, Union from sqlalchemy import types +from sqlalchemy.engine.reflection import Inspector +from sqlalchemy.sql.visitors import Visitable from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField @@ -176,6 +178,7 @@ def get_avro_for_sqlalchemy_column( def get_schema_fields_for_sqlalchemy_column( column_name: str, column_type: types.TypeEngine, + inspector: Inspector, description: Optional[str] = None, nullable: Optional[bool] = True, is_part_of_key: Optional[bool] = False, @@ -216,7 +219,10 @@ def get_schema_fields_for_sqlalchemy_column( SchemaField( fieldPath=column_name, type=SchemaFieldDataTypeClass(type=NullTypeClass()), - nativeDataType=str(column_type), + nativeDataType=get_native_data_type_for_sqlalchemy_type( + column_type, + inspector, + ), ) ] @@ -240,3 +246,25 @@ def get_schema_fields_for_sqlalchemy_column( ) return schema_fields + + +def get_native_data_type_for_sqlalchemy_type( + column_type: types.TypeEngine, inspector: Inspector +) -> str: + if isinstance(column_type, types.NullType): + return column_type.__visit_name__ + + try: + return column_type.compile(dialect=inspector.dialect) + except Exception as e: + logger.debug( + f"Unable to compile sqlalchemy type {column_type} the error was: {e}" + ) + + if ( + isinstance(column_type, Visitable) + and column_type.__visit_name__ is not None + ): + return column_type.__visit_name__ + + return repr(column_type) diff --git a/metadata-ingestion/tests/integration/hana/docker-compose.yml b/metadata-ingestion/tests/integration/hana/docker-compose.yml index 38bd1f544a0955..3f742362284835 100644 --- a/metadata-ingestion/tests/integration/hana/docker-compose.yml +++ b/metadata-ingestion/tests/integration/hana/docker-compose.yml @@ -2,15 +2,11 @@ version: '3.4' services: testhana: - image: "store/saplabs/hanaexpress:2.00.054.00.20210603.1" + image: "saplabs/hanaexpress:latest" container_name: "testhana" restart: "unless-stopped" ports: - - 39013:39013 - - 39017:39017 - - 39041-39045:39041-39045 - - 1128-1129:1128-1129 - - 59013-59014:59013-59014 + - 39041:39041 volumes: - ./post_start:/hana/hooks/post_start/ - ./setup:/hana/mounts/setup/ diff --git a/metadata-ingestion/tests/integration/hana/hana_mces_golden.json b/metadata-ingestion/tests/integration/hana/hana_mces_golden.json index 84ad1f3d3e592c..26789abc355c75 100644 --- a/metadata-ingestion/tests/integration/hana/hana_mces_golden.json +++ b/metadata-ingestion/tests/integration/hana/hana_mces_golden.json @@ -1,158 +1,217 @@ [ { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hana\", \"instance\": \"PROD\", \"database\": \"hxe\"}, \"name\": \"hxe\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "platform": "hana", + "env": "PROD", + "database": "hxe" + }, + "name": "hxe" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hana\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:hana" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hana\", \"instance\": \"PROD\", \"database\": \"hxe\", \"schema\": \"hotel\"}, \"name\": \"hotel\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "platform": "hana", + "env": "PROD", + "database": "hxe", + "schema": "hotel" + }, + "name": "hotel" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hana\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:hana" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Schema" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:c8107a53ee221a15de176e4d34a06940\"}", - "contentType": "application/json" + "json": { + "container": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "urn": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:f870c782e0a44727bd10da2ab742363b\"}", - "contentType": "application/json" + "json": { + "container": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", @@ -165,11 +224,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": {}, - "externalUrl": null, "name": "customer", - "qualifiedName": null, - "description": null, - "uri": null, "tags": [] } }, @@ -180,17 +235,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -200,189 +250,166 @@ "fields": [ { "fieldPath": "cno", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": true }, { "fieldPath": "title", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=7)", + "nativeDataType": "VARCHAR(7)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "firstname", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=20)", + "nativeDataType": "VARCHAR(20)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "zip", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=6)", + "nativeDataType": "VARCHAR(6)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Table" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "domains", "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "urn": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + }, + { + "id": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "urn": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:f870c782e0a44727bd10da2ab742363b\"}", - "contentType": "application/json" + "json": { + "container": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", @@ -395,11 +422,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": {}, - "externalUrl": null, "name": "hotel", - "qualifiedName": null, - "description": null, - "uri": null, "tags": [] } }, @@ -410,17 +433,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -430,189 +448,166 @@ "fields": [ { "fieldPath": "hno", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": true }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=30)", + "nativeDataType": "VARCHAR(30)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "state", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=2)", + "nativeDataType": "VARCHAR(2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "zip", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=6)", + "nativeDataType": "VARCHAR(6)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Table" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "domains", "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "urn": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + }, + { + "id": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "urn": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:f870c782e0a44727bd10da2ab742363b\"}", - "contentType": "application/json" + "json": { + "container": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", @@ -625,11 +620,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": {}, - "externalUrl": null, "name": "maintenance", - "qualifiedName": null, - "description": null, - "uri": null, "tags": [] } }, @@ -640,17 +631,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -660,205 +646,811 @@ "fields": [ { "fieldPath": "mno", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": true }, { "fieldPath": "hno", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "description", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "date_performed", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "performed_by", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Table" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "domains", "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "urn": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + }, + { + "id": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "urn": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 15, \"columnCount\": 6, \"fieldProfiles\": [{\"fieldPath\": \"cno\", \"uniqueCount\": 15, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"1000\", \"1001\", \"1002\", \"1003\", \"1004\", \"1005\", \"1006\", \"1007\", \"1008\", \"1009\", \"1010\", \"1011\", \"1012\", \"1013\", \"1014\"]}, {\"fieldPath\": \"title\", \"uniqueCount\": 3, \"uniqueProportion\": 0.2, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Company\", \"frequency\": 2}, {\"value\": \"Mr\", \"frequency\": 7}, {\"value\": \"Mrs\", \"frequency\": 6}], \"sampleValues\": [\"Mrs\", \"Mr\", \"Company\", \"Mrs\", \"Mrs\", \"Mr\", \"Mrs\", \"Mr\", \"Mrs\", \"Mr\", \"Mr\", \"Mrs\", \"Mr\", \"Company\", \"Mr\"]}, {\"fieldPath\": \"firstname\", \"uniqueCount\": 13, \"uniqueProportion\": 1.0, \"nullCount\": 2, \"nullProportion\": 0.13333333333333333, \"sampleValues\": [\"Jenny\", \"Peter\", \"Rose\", \"Mary\", \"Martin\", \"Sally\", \"Mike\", \"Rita\", \"George\", \"Frank\", \"Susan\", \"Joseph\", \"Antony\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 15, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Porter\", \"Brown\", \"Datasoft\", \"Brian\", \"Griffith\", \"Randolph\", \"Smith\", \"Jackson\", \"Doe\", \"Howe\", \"Miller\", \"Baker\", \"Peters\", \"TOOLware\", \"Jenkins\"]}, {\"fieldPath\": \"address\", \"uniqueCount\": 15, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"1340 N. Ash Street, #3\", \"1001 34th St., APT.3\", \"486 Maple St.\", \"500 Yellowstone Drive, #2\", \"3401 Elder Lane\", \"340 MAIN STREET, #7\", \"250 Curtis Street\", \"133 BROADWAY APT. 1\", \"2000 Humboldt St., #6\", \"111 B Parkway, #23\", \"27 5th St., 76\", \"200 MAIN STREET, #94\", \"700 S. Ash St., APT.12\", \"410 Mariposa St., #10\", \"55 A Parkway, #15\"]}, {\"fieldPath\": \"zip\", \"uniqueCount\": 12, \"uniqueProportion\": 0.8, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10580\", \"frequency\": 1}, {\"value\": \"20005\", \"frequency\": 1}, {\"value\": \"20019\", \"frequency\": 1}, {\"value\": \"20903\", \"frequency\": 1}, {\"value\": \"45211\", \"frequency\": 1}, {\"value\": \"48226\", \"frequency\": 1}, {\"value\": \"60615\", \"frequency\": 1}, {\"value\": \"75243\", \"frequency\": 3}, {\"value\": \"90018\", \"frequency\": 2}, {\"value\": \"92714\", \"frequency\": 1}, {\"value\": \"95054\", \"frequency\": 1}, {\"value\": \"97213\", \"frequency\": 1}], \"sampleValues\": [\"10580\", \"48226\", \"90018\", \"75243\", \"20005\", \"60615\", \"75243\", \"45211\", \"97213\", \"75243\", \"95054\", \"90018\", \"92714\", \"20019\", \"20903\"]}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 15, + "columnCount": 6, + "fieldProfiles": [ + { + "fieldPath": "cno", + "uniqueCount": 15, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "min": "1000", + "max": "1014", + "mean": "1007.0", + "median": "1007", + "stdev": "4.472135", + "sampleValues": [ + "1000", + "1001", + "1002", + "1003", + "1004", + "1005", + "1006", + "1007", + "1008", + "1009", + "1010", + "1011", + "1012", + "1013", + "1014" + ] + }, + { + "fieldPath": "title", + "uniqueCount": 3, + "uniqueProportion": 0.2, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "Company", + "frequency": 2 + }, + { + "value": "Mr", + "frequency": 7 + }, + { + "value": "Mrs", + "frequency": 6 + } + ], + "sampleValues": [ + "Mrs", + "Mr", + "Company", + "Mrs", + "Mrs", + "Mr", + "Mrs", + "Mr", + "Mrs", + "Mr", + "Mr", + "Mrs", + "Mr", + "Company", + "Mr" + ] + }, + { + "fieldPath": "firstname", + "uniqueCount": 13, + "uniqueProportion": 1, + "nullCount": 2, + "nullProportion": 0.13333333333333333, + "sampleValues": [ + "Jenny", + "Peter", + "Rose", + "Mary", + "Martin", + "Sally", + "Mike", + "Rita", + "George", + "Frank", + "Susan", + "Joseph", + "Antony" + ] + }, + { + "fieldPath": "name", + "uniqueCount": 15, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Porter", + "Brown", + "Datasoft", + "Brian", + "Griffith", + "Randolph", + "Smith", + "Jackson", + "Doe", + "Howe", + "Miller", + "Baker", + "Peters", + "TOOLware", + "Jenkins" + ] + }, + { + "fieldPath": "address", + "uniqueCount": 15, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "1340 N. Ash Street, #3", + "1001 34th St., APT.3", + "486 Maple St.", + "500 Yellowstone Drive, #2", + "3401 Elder Lane", + "340 MAIN STREET, #7", + "250 Curtis Street", + "133 BROADWAY APT. 1", + "2000 Humboldt St., #6", + "111 B Parkway, #23", + "27 5th St., 76", + "200 MAIN STREET, #94", + "700 S. Ash St., APT.12", + "410 Mariposa St., #10", + "55 A Parkway, #15" + ] + }, + { + "fieldPath": "zip", + "uniqueCount": 12, + "uniqueProportion": 0.8, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "10580", + "frequency": 1 + }, + { + "value": "20005", + "frequency": 1 + }, + { + "value": "20019", + "frequency": 1 + }, + { + "value": "20903", + "frequency": 1 + }, + { + "value": "45211", + "frequency": 1 + }, + { + "value": "48226", + "frequency": 1 + }, + { + "value": "60615", + "frequency": 1 + }, + { + "value": "75243", + "frequency": 3 + }, + { + "value": "90018", + "frequency": 2 + }, + { + "value": "92714", + "frequency": 1 + }, + { + "value": "95054", + "frequency": 1 + }, + { + "value": "97213", + "frequency": 1 + } + ], + "sampleValues": [ + "10580", + "48226", + "90018", + "75243", + "20005", + "60615", + "75243", + "45211", + "97213", + "75243", + "95054", + "90018", + "92714", + "20019", + "20903" + ] + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 17, \"columnCount\": 6, \"fieldProfiles\": [{\"fieldPath\": \"hno\", \"uniqueCount\": 17, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 17, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Congress\", \"Regency\", \"Long Island\", \"Empire State\", \"Midtown\", \"Eighth Avenue\", \"Lake Michigan\", \"Airport\", \"Sunshine\", \"Beach\", \"Atlantic\", \"Long Beach\", \"Indian Horse\", \"Star\", \"River Boat\", \"Ocean Star\", \"Bella Ciente\"]}, {\"fieldPath\": \"address\", \"uniqueCount\": 17, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"155 Beechwood St.\", \"477 17th Avenue\", \"1499 Grove Street\", \"65 Yellowstone Dr.\", \"12 Barnard St.\", \"112 8th Avenue\", \"354 OAK Terrace\", \"650 C Parkway\", \"200 Yellowstone Dr.\", \"1980 34th St.\", \"111 78th St.\", \"35 Broadway\", \"16 MAIN STREET\", \"13 Beechwood Place\", \"788 MAIN STREET\", \"45 Pacific Avenue\", \"1407 Marshall Ave\"]}, {\"fieldPath\": \"city\", \"uniqueCount\": 15, \"uniqueProportion\": 0.8823529411764706, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Albany\", \"frequency\": 1}, {\"value\": \"Atlantic City\", \"frequency\": 1}, {\"value\": \"Chicago\", \"frequency\": 1}, {\"value\": \"Clearwater\", \"frequency\": 1}, {\"value\": \"Daytona Beach\", \"frequency\": 1}, {\"value\": \"Deerfield Beach\", \"frequency\": 1}, {\"value\": \"Hollywood\", \"frequency\": 1}, {\"value\": \"Long Beach\", \"frequency\": 1}, {\"value\": \"Long Island\", \"frequency\": 1}, {\"value\": \"Longview\", \"frequency\": 1}, {\"value\": \"New Orleans\", \"frequency\": 1}, {\"value\": \"New York\", \"frequency\": 2}, {\"value\": \"Palm Springs\", \"frequency\": 1}, {\"value\": \"Rosemont\", \"frequency\": 1}, {\"value\": \"Seattle\", \"frequency\": 2}], \"sampleValues\": [\"Seattle\", \"Seattle\", \"Long Island\", \"Albany\", \"New York\", \"New York\", \"Chicago\", \"Rosemont\", \"Clearwater\", \"Daytona Beach\", \"Deerfield Beach\", \"Long Beach\", \"Palm Springs\", \"Hollywood\", \"New Orleans\", \"Atlantic City\", \"Longview\"]}, {\"fieldPath\": \"state\", \"uniqueCount\": 8, \"uniqueProportion\": 0.47058823529411764, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"CA\", \"frequency\": 3}, {\"value\": \"FL\", \"frequency\": 3}, {\"value\": \"IL\", \"frequency\": 2}, {\"value\": \"LA\", \"frequency\": 1}, {\"value\": \"NJ\", \"frequency\": 1}, {\"value\": \"NY\", \"frequency\": 4}, {\"value\": \"TX\", \"frequency\": 1}, {\"value\": \"WA\", \"frequency\": 2}], \"sampleValues\": [\"WA\", \"WA\", \"NY\", \"NY\", \"NY\", \"NY\", \"IL\", \"IL\", \"FL\", \"FL\", \"FL\", \"CA\", \"CA\", \"CA\", \"LA\", \"NJ\", \"TX\"]}, {\"fieldPath\": \"zip\", \"uniqueCount\": 16, \"uniqueProportion\": 0.9411764705882353, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"08401\", \"frequency\": 1}, {\"value\": \"10019\", \"frequency\": 2}, {\"value\": \"11788\", \"frequency\": 1}, {\"value\": \"12203\", \"frequency\": 1}, {\"value\": \"20005\", \"frequency\": 1}, {\"value\": \"20037\", \"frequency\": 1}, {\"value\": \"32018\", \"frequency\": 1}, {\"value\": \"33441\", \"frequency\": 1}, {\"value\": \"33575\", \"frequency\": 1}, {\"value\": \"60018\", \"frequency\": 1}, {\"value\": \"60601\", \"frequency\": 1}, {\"value\": \"70112\", \"frequency\": 1}, {\"value\": \"75601\", \"frequency\": 1}, {\"value\": \"90029\", \"frequency\": 1}, {\"value\": \"90804\", \"frequency\": 1}, {\"value\": \"92262\", \"frequency\": 1}], \"sampleValues\": [\"20005\", \"20037\", \"11788\", \"12203\", \"10019\", \"10019\", \"60601\", \"60018\", \"33575\", \"32018\", \"33441\", \"90804\", \"92262\", \"90029\", \"70112\", \"08401\", \"75601\"]}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 17, + "columnCount": 6, + "fieldProfiles": [ + { + "fieldPath": "hno", + "uniqueCount": 17, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "min": "10", + "max": "26", + "mean": "18.0", + "median": "18", + "stdev": "5.049752", + "sampleValues": [ + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26" + ] + }, + { + "fieldPath": "name", + "uniqueCount": 17, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Congress", + "Regency", + "Long Island", + "Empire State", + "Midtown", + "Eighth Avenue", + "Lake Michigan", + "Airport", + "Sunshine", + "Beach", + "Atlantic", + "Long Beach", + "Indian Horse", + "Star", + "River Boat", + "Ocean Star", + "Bella Ciente" + ] + }, + { + "fieldPath": "address", + "uniqueCount": 17, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "155 Beechwood St.", + "477 17th Avenue", + "1499 Grove Street", + "65 Yellowstone Dr.", + "12 Barnard St.", + "112 8th Avenue", + "354 OAK Terrace", + "650 C Parkway", + "200 Yellowstone Dr.", + "1980 34th St.", + "111 78th St.", + "35 Broadway", + "16 MAIN STREET", + "13 Beechwood Place", + "788 MAIN STREET", + "45 Pacific Avenue", + "1407 Marshall Ave" + ] + }, + { + "fieldPath": "city", + "uniqueCount": 15, + "uniqueProportion": 0.8823529411764706, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "Albany", + "frequency": 1 + }, + { + "value": "Atlantic City", + "frequency": 1 + }, + { + "value": "Chicago", + "frequency": 1 + }, + { + "value": "Clearwater", + "frequency": 1 + }, + { + "value": "Daytona Beach", + "frequency": 1 + }, + { + "value": "Deerfield Beach", + "frequency": 1 + }, + { + "value": "Hollywood", + "frequency": 1 + }, + { + "value": "Long Beach", + "frequency": 1 + }, + { + "value": "Long Island", + "frequency": 1 + }, + { + "value": "Longview", + "frequency": 1 + }, + { + "value": "New Orleans", + "frequency": 1 + }, + { + "value": "New York", + "frequency": 2 + }, + { + "value": "Palm Springs", + "frequency": 1 + }, + { + "value": "Rosemont", + "frequency": 1 + }, + { + "value": "Seattle", + "frequency": 2 + } + ], + "sampleValues": [ + "Seattle", + "Seattle", + "Long Island", + "Albany", + "New York", + "New York", + "Chicago", + "Rosemont", + "Clearwater", + "Daytona Beach", + "Deerfield Beach", + "Long Beach", + "Palm Springs", + "Hollywood", + "New Orleans", + "Atlantic City", + "Longview" + ] + }, + { + "fieldPath": "state", + "uniqueCount": 8, + "uniqueProportion": 0.47058823529411764, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "CA", + "frequency": 3 + }, + { + "value": "FL", + "frequency": 3 + }, + { + "value": "IL", + "frequency": 2 + }, + { + "value": "LA", + "frequency": 1 + }, + { + "value": "NJ", + "frequency": 1 + }, + { + "value": "NY", + "frequency": 4 + }, + { + "value": "TX", + "frequency": 1 + }, + { + "value": "WA", + "frequency": 2 + } + ], + "sampleValues": [ + "WA", + "WA", + "NY", + "NY", + "NY", + "NY", + "IL", + "IL", + "FL", + "FL", + "FL", + "CA", + "CA", + "CA", + "LA", + "NJ", + "TX" + ] + }, + { + "fieldPath": "zip", + "uniqueCount": 16, + "uniqueProportion": 0.9411764705882353, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "08401", + "frequency": 1 + }, + { + "value": "10019", + "frequency": 2 + }, + { + "value": "11788", + "frequency": 1 + }, + { + "value": "12203", + "frequency": 1 + }, + { + "value": "20005", + "frequency": 1 + }, + { + "value": "20037", + "frequency": 1 + }, + { + "value": "32018", + "frequency": 1 + }, + { + "value": "33441", + "frequency": 1 + }, + { + "value": "33575", + "frequency": 1 + }, + { + "value": "60018", + "frequency": 1 + }, + { + "value": "60601", + "frequency": 1 + }, + { + "value": "70112", + "frequency": 1 + }, + { + "value": "75601", + "frequency": 1 + }, + { + "value": "90029", + "frequency": 1 + }, + { + "value": "90804", + "frequency": 1 + }, + { + "value": "92262", + "frequency": 1 + } + ], + "sampleValues": [ + "20005", + "20037", + "11788", + "12203", + "10019", + "10019", + "60601", + "60018", + "33575", + "32018", + "33441", + "90804", + "92262", + "90029", + "70112", + "08401", + "75601" + ] + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 3, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"mno\", \"uniqueCount\": 3, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"10\", \"11\", \"12\"]}, {\"fieldPath\": \"hno\", \"uniqueCount\": 3, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"24\", \"25\", \"26\"]}, {\"fieldPath\": \"description\", \"uniqueCount\": 3, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Replace pool liner and pump\", \"Renovate the bar area. Replace TV and speakers\", \"Roof repair due to storm\"]}, {\"fieldPath\": \"date_performed\", \"uniqueCount\": 2, \"uniqueProportion\": 1.0, \"nullCount\": 1, \"nullProportion\": 0.3333333333333333, \"min\": \"2019-03-21\", \"max\": \"2020-11-29\", \"sampleValues\": [\"2019-03-21\", \"2020-11-29\"]}, {\"fieldPath\": \"performed_by\", \"uniqueCount\": 2, \"uniqueProportion\": 1.0, \"nullCount\": 1, \"nullProportion\": 0.3333333333333333, \"sampleValues\": [\"Discount Pool Supplies\", \"TV and Audio Superstore\"]}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 3, + "columnCount": 5, + "fieldProfiles": [ + { + "fieldPath": "mno", + "uniqueCount": 3, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "min": "10", + "max": "12", + "mean": "11.0", + "median": "11", + "stdev": "1.0", + "sampleValues": [ + "10", + "11", + "12" + ] + }, + { + "fieldPath": "hno", + "uniqueCount": 3, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "min": "24", + "max": "26", + "mean": "25.0", + "median": "25", + "stdev": "1.0", + "sampleValues": [ + "24", + "25", + "26" + ] + }, + { + "fieldPath": "description", + "uniqueCount": 3, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Replace pool liner and pump", + "Renovate the bar area. Replace TV and speakers", + "Roof repair due to storm" + ] + }, + { + "fieldPath": "date_performed", + "uniqueCount": 2, + "uniqueProportion": 1, + "nullCount": 1, + "nullProportion": 0.3333333333333333, + "min": "2019-03-21", + "max": "2020-11-29", + "sampleValues": [ + "2019-03-21", + "2020-11-29" + ] + }, + { + "fieldPath": "performed_by", + "uniqueCount": 2, + "uniqueProportion": 1, + "nullCount": 1, + "nullProportion": 0.3333333333333333, + "sampleValues": [ + "Discount Pool Supplies", + "TV and Audio Superstore" + ] + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/expected_output.json b/metadata-ingestion/tests/integration/lookml/expected_output.json index d870c6dee40655..f42c600281ccb3 100644 --- a/metadata-ingestion/tests/integration/lookml/expected_output.json +++ b/metadata-ingestion/tests/integration/lookml/expected_output.json @@ -1632,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n order.region='ap-south-1'\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", "viewLanguage": "sql" } }, diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json index 8813ea532fa2b5..5f9b99ebe30623 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json @@ -1632,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n order.region='ap-south-1'\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", "viewLanguage": "sql" } }, diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json index 4bc1a0f2f7da58..1b95959f0ba1d2 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json @@ -1632,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n order.region='ap-south-1'\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", "viewLanguage": "sql" } }, diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json index 3fd37c47221858..fd479a2baa7226 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json @@ -1675,6 +1675,96 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),sale_price)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "customer_facts", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lifetime_spend", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json index 3fd37c47221858..fd479a2baa7226 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json @@ -1675,6 +1675,96 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),sale_price)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "customer_facts", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lifetime_spend", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json index bb8a379fdde224..053e90d473c1ba 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json @@ -1675,6 +1675,96 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.order,DEV)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.order,DEV),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.order,DEV),sale_price)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "customer_facts", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lifetime_spend", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json index b8a2bcc020c34d..44dd72e8fc41be 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json @@ -1683,6 +1683,96 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD),sale_price)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "customer_facts", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lifetime_spend", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json index 7265ee3c6c62b9..7c2f92ac1e028c 100644 --- a/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json +++ b/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json @@ -1656,7 +1656,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n order.region='ap-south-1'\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", "viewLanguage": "sql" } }, diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml index ea55512c5ca06e..6eb92d749c9f7f 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml @@ -5,6 +5,7 @@ include: "employee_income_source.view.lkml" include: "employee_total_income.view.lkml" include: "top_10_employee_income_source.view.lkml" include: "employee_tax_report.view.lkml" +include: "employee_salary_rating.view.lkml" explore: activity_logs { } @@ -19,4 +20,7 @@ explore: top_10_employee_income_source { } explore: employee_tax_report { +} + +explore: employee_salary_rating { } \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_salary_rating.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_salary_rating.view.lkml new file mode 100644 index 00000000000000..3a00099e7998e9 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_salary_rating.view.lkml @@ -0,0 +1,50 @@ +view: employee_salary_rating { + derived_table: { + sql: SELECT + employee_id, + employee_name, + {% if dw_eff_dt_date._is_selected or finance_dw_eff_dt_date._is_selected %} + prod_core.data.r_metric_summary_v2 + {% elsif dw_eff_dt_week._is_selected or finance_dw_eff_dt_week._is_selected %} + prod_core.data.r_metric_summary_v3 + {% else %} + 'default_table' as source + {% endif %}, + employee_income + FROM source_table + WHERE + {% condition source_region %} source_table.region {% endcondition %} AND + {% if rating_window._is_filtered %} + {% condition rating_window %} DATE (rating_created) {% endcondition %} + {% endif %} + ;; + } + + filter: rating_window { + description: "Date window in which to look for rating" + default_value: "90 days ago for 90 days" + datatype: date + type: date + } + + dimension: id { + type: number + sql: ${TABLE}.employee_id;; + } + + dimension: name { + type: string + sql: ${TABLE}.employee_name;; + } + + dimension: source { + type: string + sql: ${TABLE}.source ;; + } + + dimension: income { + type: number + sql: ${TABLE}.employee_income ;; + } + +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json index 75cd50c5c6059e..d12ced5e425066 100644 --- a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json @@ -302,7 +302,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n employee_id,\n employee_name,\n \n prod_core.data.r_metric_summary_v2\n ,\n employee_income\n FROM source_table\n WHERE\n source_table.region='ap-south-1'", + "viewLogic": "SELECT\n employee_id,\n employee_name,\n {% if dw_eff_dt_date._is_selected or finance_dw_eff_dt_date._is_selected %}\n prod_core.data.r_metric_summary_v2\n {% elsif dw_eff_dt_week._is_selected or finance_dw_eff_dt_week._is_selected %}\n prod_core.data.r_metric_summary_v3\n {% else %}\n 'default_table' as source\n {% endif %},\n employee_income\n FROM source_table\n WHERE\n {% condition source_region %} source_table.region {% endcondition %}", "viewLanguage": "sql" } }, @@ -1300,6 +1300,286 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT\n employee_id,\n employee_name,\n {% if dw_eff_dt_date._is_selected or finance_dw_eff_dt_date._is_selected %}\n prod_core.data.r_metric_summary_v2\n {% elsif dw_eff_dt_week._is_selected or finance_dw_eff_dt_week._is_selected %}\n prod_core.data.r_metric_summary_v3\n {% else %}\n 'default_table' as source\n {% endif %},\n employee_income\n FROM source_table\n WHERE\n {% condition source_region %} source_table.region {% endcondition %} AND\n {% if rating_window._is_filtered %}\n {% condition rating_window %} DATE (rating_created) {% endcondition %}\n {% endif %}", + "viewLanguage": "sql" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD),employee_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD),employee_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD),name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD),source)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD),source)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD),employee_income)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD),income)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "employee_salary_rating", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "income", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "employee_salary_rating.view.lkml", + "looker.model": "data" + }, + "name": "employee_salary_rating", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_salary_rating,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json index f78b45fb373fc6..bddccc856c842c 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json @@ -147,7 +147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -159,7 +159,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -171,7 +171,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=14)", + "nativeDataType": "VARCHAR(14)", "recursive": false, "isPartOfKey": false }, @@ -183,7 +183,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -195,7 +195,7 @@ "com.linkedin.pegasus2avro.schema.EnumType": {} } }, - "nativeDataType": "ENUM('M', 'F')", + "nativeDataType": "ENUM('M','F')", "recursive": false, "glossaryTerms": { "terms": [ @@ -218,7 +218,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -334,7 +334,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -346,7 +346,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -358,7 +358,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": true }, @@ -370,7 +370,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -439,7 +439,6 @@ }, "rowCount": 10, "columnCount": 6, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "emp_no", @@ -574,7 +573,8 @@ "1989-08-24" ] } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -597,7 +597,6 @@ }, "rowCount": 112, "columnCount": 4, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "emp_no", @@ -991,7 +990,8 @@ "1993-02-09" ] } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -1166,7 +1166,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "isPartOfKey": true }, @@ -1178,7 +1178,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": true }, @@ -1190,7 +1190,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": true }, @@ -1202,7 +1202,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "LONGTEXT()", + "nativeDataType": "LONGTEXT", "recursive": false, "isPartOfKey": false }, @@ -1214,7 +1214,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "DATETIME(fsp=6)", + "nativeDataType": "DATETIME(6)", "recursive": false, "isPartOfKey": false }, @@ -1226,7 +1226,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false }, @@ -1238,7 +1238,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false } @@ -1373,7 +1373,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": true }, @@ -1386,7 +1386,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false }, @@ -1398,7 +1398,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=150)", + "nativeDataType": "VARCHAR(150)", "recursive": false, "isPartOfKey": false }, @@ -1410,7 +1410,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=150)", + "nativeDataType": "VARCHAR(150)", "recursive": false, "isPartOfKey": false }, @@ -1422,7 +1422,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": false }, @@ -1434,7 +1434,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false }, @@ -1446,7 +1446,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DOUBLE(asdecimal=True)", + "nativeDataType": "DOUBLE", "recursive": false, "isPartOfKey": false } @@ -1583,7 +1583,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": false }, @@ -1595,7 +1595,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false }, @@ -1607,7 +1607,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=150)", + "nativeDataType": "VARCHAR(150)", "recursive": false, "isPartOfKey": false }, @@ -1619,7 +1619,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DOUBLE(asdecimal=True)", + "nativeDataType": "DOUBLE", "recursive": false, "isPartOfKey": false } @@ -1858,7 +1858,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1870,7 +1870,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -1882,7 +1882,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -1894,7 +1894,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -1906,7 +1906,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "glossaryTerms": { "terms": [ @@ -1929,7 +1929,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -2045,7 +2045,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -2057,7 +2057,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -2069,7 +2069,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -2150,7 +2150,6 @@ }, "rowCount": 5, "columnCount": 6, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "id", @@ -2259,7 +2258,8 @@ "3.8" ] } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -2282,7 +2282,6 @@ }, "rowCount": 0, "columnCount": 3, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "id", @@ -2299,7 +2298,8 @@ "uniqueCount": 0, "nullCount": 0 } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -2456,7 +2456,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "SET('a', 'b', 'c', 'd')", + "nativeDataType": "SET('a','b','c','d')", "recursive": false, "isPartOfKey": false } @@ -2572,7 +2572,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false } @@ -2641,14 +2641,14 @@ }, "rowCount": 0, "columnCount": 1, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "col", "uniqueCount": 0, "nullCount": 0 } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -2671,14 +2671,14 @@ }, "rowCount": 0, "columnCount": 1, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "dummy", "uniqueCount": 0, "nullCount": 0 } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json index 065d6cbe90b313..8c6f6338bc2b07 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json @@ -165,7 +165,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -177,7 +177,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -189,7 +189,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -201,7 +201,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -213,7 +213,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -225,7 +225,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -359,7 +359,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -371,7 +371,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -383,7 +383,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json index b8dfd7d9efc37d..3f5899aa8a98c3 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json @@ -147,7 +147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -159,7 +159,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -171,7 +171,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -183,7 +183,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -195,7 +195,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -207,7 +207,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -323,7 +323,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -335,7 +335,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -347,7 +347,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -461,63 +461,5 @@ "runId": "mysql-2020_04_14-07_00_00", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", - "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", - "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json index fc25af0400bb5d..58a70cae2b2d5e 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json @@ -147,7 +147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -159,7 +159,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -171,7 +171,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -183,7 +183,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -195,7 +195,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -207,7 +207,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -323,7 +323,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -335,7 +335,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -347,7 +347,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json index b740dfe025ef7f..6732b17d2e8322 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json @@ -258,7 +258,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -380,7 +380,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -505,7 +505,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": false } @@ -754,7 +754,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -876,7 +876,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -1001,7 +1001,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": false } @@ -1207,213 +1207,5 @@ "runId": "oracle-2022_02_03-07_00_00", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.test1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.test2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.test3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.test4,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json index 008cd405186c39..7610daaa54b4a1 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json @@ -258,7 +258,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -380,7 +380,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -505,7 +505,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": false } @@ -754,7 +754,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -876,7 +876,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -1001,7 +1001,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": false } @@ -1053,214 +1053,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.test1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.test2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.test3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.test4,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD)", diff --git a/metadata-ingestion/tests/integration/oracle/test_oracle.py b/metadata-ingestion/tests/integration/oracle/test_oracle.py index 6c9aba8ec5620e..4541bb8ac65bff 100644 --- a/metadata-ingestion/tests/integration/oracle/test_oracle.py +++ b/metadata-ingestion/tests/integration/oracle/test_oracle.py @@ -24,6 +24,7 @@ def apply_mock_data(self, mock_create_engine, mock_inspect, mock_event): inspector_magic_mock.dialect.server_version_info = ( self.get_server_version_info() ) + inspector_magic_mock.dialect.type_compiler.process = lambda x: "NUMBER" mock_inspect.return_value = inspector_magic_mock mock_create_engine.connect.return_value = connection_magic_mock diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json index f35ff9fdb9d153..f42ff7c0df068c 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json @@ -452,7 +452,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "isPartOfKey": true }, @@ -464,7 +464,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": true }, @@ -476,7 +476,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": true }, @@ -488,7 +488,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TEXT()", + "nativeDataType": "TEXT", "recursive": false, "isPartOfKey": false }, @@ -500,7 +500,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TEXT()", + "nativeDataType": "TEXT", "recursive": false, "isPartOfKey": false }, @@ -512,7 +512,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP()", + "nativeDataType": "TIMESTAMP WITHOUT TIME ZONE", "recursive": false, "isPartOfKey": false }, @@ -524,7 +524,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false }, @@ -536,7 +536,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false }, @@ -548,7 +548,7 @@ "com.linkedin.pegasus2avro.schema.RecordType": {} } }, - "nativeDataType": "JSON(astext_type=Text())", + "nativeDataType": "JSON", "recursive": false, "isPartOfKey": false } @@ -671,7 +671,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "isPartOfKey": false }, @@ -683,7 +683,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json index f47789fc470cd8..f107fb1006bf6e 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json @@ -256,7 +256,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "glossaryTerms": { "terms": [ @@ -279,7 +279,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": true }, @@ -291,7 +291,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": true }, @@ -303,7 +303,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TEXT()", + "nativeDataType": "TEXT", "recursive": false, "isPartOfKey": false }, @@ -315,7 +315,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TEXT()", + "nativeDataType": "TEXT", "recursive": false, "isPartOfKey": false }, @@ -327,7 +327,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP()", + "nativeDataType": "TIMESTAMP WITHOUT TIME ZONE", "recursive": false, "isPartOfKey": false }, @@ -339,7 +339,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "glossaryTerms": { "terms": [ @@ -362,7 +362,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false }, @@ -374,7 +374,7 @@ "com.linkedin.pegasus2avro.schema.RecordType": {} } }, - "nativeDataType": "JSON(astext_type=Text())", + "nativeDataType": "JSON", "recursive": false, "isPartOfKey": false } @@ -497,7 +497,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "isPartOfKey": false }, @@ -509,7 +509,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json index 8ec431b6fe9f1b..049008bddc58ed 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json @@ -312,6 +312,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index 29e9ccebf067ec..fa4bcb8abaa94d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -1226,6 +1226,40 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, @@ -1951,6 +1985,50 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 937cad0b9ec17e..60b36897ed2e42 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -1276,6 +1276,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index 501ec284097b39..b43e4a6c2c1c2d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -1721,6 +1721,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, @@ -2903,6 +2942,60 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e", @@ -3065,6 +3158,30 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C22-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard2" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 74779ac7a0577e..c5414444cc35b8 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -986,6 +986,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index 442dfd5c8c0825..e1ddbfb901badd 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -1158,6 +1158,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 2c4ff6ee851f4a..6f899a7fa11b72 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -1018,6 +1018,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json new file mode 100644 index 00000000000000..efbd9abfdb9118 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json @@ -0,0 +1,1153 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "public issue_history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "SNOWFLAKE_TESTTABLE", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "big-query-with-parameter", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User1@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User2@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Tile" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Tile" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + }, + "title": "test_dashboard", + "description": "Description of test dashboard", + "charts": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "https://localhost/dashboards/web/1" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" + }, + { + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index 0aea8514559ecb..9a09cb4fec64d0 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -1201,6 +1201,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index 22a00236af8c61..d80aa02c4cb123 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -1018,6 +1018,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json index 11a7fed6030cec..66e87952bf1416 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json @@ -1028,6 +1028,40 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, @@ -1257,6 +1291,50 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index cf5d4df460e231..ea1ee0df4b1057 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -1026,6 +1026,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index cce9e3f8755ba5..094869bfd24f1d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -1018,6 +1018,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, @@ -1935,6 +1974,60 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index 5e244e0e0f93f7..dcaa518a3c3237 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -986,6 +986,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, @@ -1087,6 +1126,30 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C22-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard2" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index 8fd8989b81122b..bc5e844f679c71 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -1226,6 +1226,45 @@ "op": "add", "path": "/customProperties/workspaceId", "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index 06cc40fe7b24c1..6a95ec2c1dda42 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -683,6 +683,52 @@ def test_powerbi_ingest( ) +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +@pytest.mark.integration +def test_powerbi_ingest_patch_disabled( + mock_msal: MagicMock, + pytestconfig: pytest.Config, + tmp_path: str, + mock_time: datetime.datetime, + requests_mock: Any, +) -> None: + enable_logging() + + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + register_mock_api(request_mock=requests_mock) + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + "patch_metadata": False, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_mces.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_ingest_patch_disabled.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/powerbi_mces.json", + golden_path=f"{test_resources_dir}/{golden_file}", + ) + + @freeze_time(FROZEN_TIME) @mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) @pytest.mark.integration diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 4c0c1c6512ec77..74cb216117bd43 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -112,11 +112,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", + "job_id": "c6fb6778-14f1-4516-bb41-e5eaa97a687b", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-01-19 11:45:06.667000", - "date_modified": "2024-01-19 11:45:06.840000", + "date_created": "2024-07-27 23:58:29.780000", + "date_modified": "2024-07-27 23:58:29.943000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1304,7 +1304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1316,7 +1316,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1546,7 +1546,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1558,7 +1558,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1678,7 +1678,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1691,7 +1691,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1703,7 +1703,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1715,7 +1715,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1835,7 +1835,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1847,7 +1847,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "UNIQUEIDENTIFIER()", + "nativeDataType": "UNIQUEIDENTIFIER", "recursive": false, "isPartOfKey": false }, @@ -1859,7 +1859,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR(length=50)", + "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-01-19 11:45:06.590000", - "date_modified": "2024-01-19 11:45:06.590000" + "date_created": "2024-07-27 23:58:29.703000", + "date_modified": "2024-07-27 23:58:29.703000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -3560,7 +3560,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -3572,7 +3572,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -3584,7 +3584,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "MONEY()", + "nativeDataType": "MONEY", "recursive": false, "isPartOfKey": false } @@ -3813,7 +3813,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -3825,7 +3825,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -3837,7 +3837,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "SMALLMONEY()", + "nativeDataType": "SMALLMONEY", "recursive": false, "isPartOfKey": false } @@ -3957,7 +3957,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -3969,7 +3969,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -3981,7 +3981,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -3993,7 +3993,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -4442,800 +4442,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:47217386c89d8b94153f6ee31e7e77ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5eb0d61efa998d1ccd5cbdc6ce4bb4af", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2816b2cb7f90d3dce64125ba89fb1fa8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:20d0f0c94e9796ff44ff32d4d0e19084", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3600d2ebb33b25dac607624d7eae7575", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:280f2e3aefacc346d0ce1590ec337c7d", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:cba5c3ca7f028fcf749593be369d3c24", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:58c30fa72f213ca7e12fb04f5a7d150f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9387ddfeb7b57672cabd761ade89c49c", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.dbo.ProductsNew,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - }, - { - "id": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", - "urn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.ItemsNew,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - }, - { - "id": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", - "urn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.PersonsNew,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - }, - { - "id": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", - "urn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:54727d9fd7deacef27641559125bbc56", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:141b0980dcb08f48544583e47cf48807", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 02c357259c3f53..e1af3f72a8af1f 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -112,11 +112,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", + "job_id": "c6fb6778-14f1-4516-bb41-e5eaa97a687b", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-01-19 11:45:06.667000", - "date_modified": "2024-01-19 11:45:06.840000", + "date_created": "2024-07-27 23:58:29.780000", + "date_modified": "2024-07-27 23:58:29.943000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1304,7 +1304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1316,7 +1316,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1546,7 +1546,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1558,7 +1558,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1678,7 +1678,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1691,7 +1691,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1703,7 +1703,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1715,7 +1715,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1835,7 +1835,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1847,7 +1847,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "UNIQUEIDENTIFIER()", + "nativeDataType": "UNIQUEIDENTIFIER", "recursive": false, "isPartOfKey": false }, @@ -1859,7 +1859,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR(length=50)", + "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-01-19 11:45:06.590000", - "date_modified": "2024-01-19 11:45:06.590000" + "date_created": "2024-07-27 23:58:29.703000", + "date_modified": "2024-07-27 23:58:29.703000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -2385,415 +2385,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 02c357259c3f53..e1af3f72a8af1f 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -112,11 +112,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", + "job_id": "c6fb6778-14f1-4516-bb41-e5eaa97a687b", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-01-19 11:45:06.667000", - "date_modified": "2024-01-19 11:45:06.840000", + "date_created": "2024-07-27 23:58:29.780000", + "date_modified": "2024-07-27 23:58:29.943000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1304,7 +1304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1316,7 +1316,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1546,7 +1546,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1558,7 +1558,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1678,7 +1678,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1691,7 +1691,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1703,7 +1703,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1715,7 +1715,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1835,7 +1835,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1847,7 +1847,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "UNIQUEIDENTIFIER()", + "nativeDataType": "UNIQUEIDENTIFIER", "recursive": false, "isPartOfKey": false }, @@ -1859,7 +1859,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR(length=50)", + "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-01-19 11:45:06.590000", - "date_modified": "2024-01-19 11:45:06.590000" + "date_created": "2024-07-27 23:58:29.703000", + "date_modified": "2024-07-27 23:58:29.703000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -2385,415 +2385,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index ad15c654e44c96..5b936c0d9f2446 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -112,11 +112,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", + "job_id": "c6fb6778-14f1-4516-bb41-e5eaa97a687b", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-01-19 11:45:06.667000", - "date_modified": "2024-01-19 11:45:06.840000", + "date_created": "2024-07-27 23:58:29.780000", + "date_modified": "2024-07-27 23:58:29.943000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1304,7 +1304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1316,7 +1316,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1546,7 +1546,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1558,7 +1558,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1678,7 +1678,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1691,7 +1691,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1703,7 +1703,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1715,7 +1715,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1835,7 +1835,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1847,7 +1847,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "UNIQUEIDENTIFIER()", + "nativeDataType": "UNIQUEIDENTIFIER", "recursive": false, "isPartOfKey": false }, @@ -1859,7 +1859,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR(length=50)", + "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-01-19 11:45:06.590000", - "date_modified": "2024-01-19 11:45:06.590000" + "date_created": "2024-07-27 23:58:29.703000", + "date_modified": "2024-07-27 23:58:29.703000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -2385,415 +2385,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json index c5664b9373e8c5..d19b83f4d4e2f8 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json @@ -244,7 +244,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1710150034" + "transient_lastddltime": "1722106707" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -280,7 +280,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -505,7 +505,7 @@ "numrows": "3", "rawdatasize": "94", "totalsize": "97", - "transient_lastddltime": "1710150038" + "transient_lastddltime": "1722106711" }, "name": "classification_test", "tags": [] @@ -539,7 +539,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -551,7 +551,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -563,7 +563,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -575,7 +575,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -587,7 +587,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -764,7 +764,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150036" + "transient_lastddltime": "1722106709" }, "name": "map_test", "tags": [] @@ -798,7 +798,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -991,7 +991,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150036" + "transient_lastddltime": "1722106709" }, "name": "nested_struct_test", "tags": [] @@ -1025,7 +1025,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1262,7 +1262,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1710150028" + "transient_lastddltime": "1722106702" }, "name": "pokes", "tags": [] @@ -1296,7 +1296,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1308,7 +1308,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -1320,7 +1320,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false } @@ -1497,7 +1497,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150031" + "transient_lastddltime": "1722106704" }, "name": "struct_test", "tags": [] @@ -1531,7 +1531,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1748,7 +1748,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1710150036" + "transient_lastddltime": "1722106709" }, "name": "struct_test_view_materialized", "tags": [] @@ -1782,7 +1782,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2002,7 +2002,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150031" + "transient_lastddltime": "1722106704" }, "name": "_test_table_underscore", "tags": [] @@ -2036,7 +2036,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2048,7 +2048,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false } @@ -2225,7 +2225,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150036" + "transient_lastddltime": "1722106709" }, "name": "union_test", "tags": [] @@ -2527,7 +2527,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1710150036", + "transient_lastddltime": "1722106709", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, @@ -2563,7 +2563,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json index 2764433808cbdb..f72610fba7c547 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json @@ -231,7 +231,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1713211020" + "transient_lastddltime": "1722106707" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -267,7 +267,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -471,7 +471,7 @@ "numrows": "3", "rawdatasize": "94", "totalsize": "97", - "transient_lastddltime": "1713211025" + "transient_lastddltime": "1722106711" }, "name": "classification_test", "tags": [] @@ -505,7 +505,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -517,7 +517,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "glossaryTerms": { "terms": [ @@ -540,7 +540,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "glossaryTerms": { "terms": [ @@ -563,7 +563,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "glossaryTerms": { "terms": [ @@ -586,7 +586,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "glossaryTerms": { "terms": [ @@ -753,7 +753,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211023" + "transient_lastddltime": "1722106709" }, "name": "map_test", "tags": [] @@ -787,7 +787,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -959,7 +959,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211023" + "transient_lastddltime": "1722106709" }, "name": "nested_struct_test", "tags": [] @@ -993,7 +993,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1209,7 +1209,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1713211015" + "transient_lastddltime": "1722106702" }, "name": "pokes", "tags": [] @@ -1243,7 +1243,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1255,7 +1255,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -1267,7 +1267,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false } @@ -1423,7 +1423,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211017" + "transient_lastddltime": "1722106704" }, "name": "struct_test", "tags": [] @@ -1457,7 +1457,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1653,7 +1653,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1713211023" + "transient_lastddltime": "1722106709" }, "name": "struct_test_view_materialized", "tags": [] @@ -1687,7 +1687,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1886,7 +1886,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211017" + "transient_lastddltime": "1722106704" }, "name": "_test_table_underscore", "tags": [] @@ -1920,7 +1920,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1932,7 +1932,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false } @@ -2088,7 +2088,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211023" + "transient_lastddltime": "1722106709" }, "name": "union_test", "tags": [] @@ -2369,7 +2369,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1713211023", + "transient_lastddltime": "1722106709", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, @@ -2405,7 +2405,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, diff --git a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json index 1f03f02fa9408f..b2afad81b12fad 100644 --- a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json @@ -256,7 +256,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -268,7 +268,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -280,7 +280,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -292,7 +292,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -304,7 +304,7 @@ "com.linkedin.pegasus2avro.schema.RecordType": {} } }, - "nativeDataType": "JSON()", + "nativeDataType": "JSON", "recursive": false, "isPartOfKey": false }, @@ -504,7 +504,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -516,7 +516,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -528,7 +528,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -540,7 +540,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -723,7 +723,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -735,7 +735,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false } @@ -918,7 +918,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -930,7 +930,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -942,7 +942,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -954,7 +954,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -966,7 +966,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -978,7 +978,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -1479,142 +1479,5 @@ "runId": "trino-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,postgresqldb.librarydb.book,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - }, - { - "id": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "urn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,postgresqldb.librarydb.issue_history,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - }, - { - "id": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "urn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,postgresqldb.librarydb.member,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - }, - { - "id": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "urn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,postgresqldb.librarydb.book_in_circulation,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - }, - { - "id": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "urn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json index cd1cd0d7e28a48..b7e0f268aa7fe1 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json @@ -250,7 +250,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:31.057395+00:00", + "create_time": "2024-06-18 12:46:11.762979+00:00", "table_size": "0 KB" }, "name": "clicks", @@ -287,7 +287,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -300,7 +300,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -313,7 +313,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false } @@ -427,7 +427,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.887434+00:00", + "create_time": "2024-06-18 12:45:49.748139+00:00", "table_size": "2119 KB" }, "name": "customer_dimension", @@ -464,7 +464,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -477,7 +477,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -490,7 +490,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -503,7 +503,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -516,7 +516,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -529,7 +529,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -542,7 +542,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -555,7 +555,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -568,7 +568,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -581,7 +581,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -594,7 +594,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -607,7 +607,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -620,7 +620,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -633,7 +633,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -646,7 +646,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -659,7 +659,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -672,7 +672,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -685,7 +685,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -698,7 +698,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -711,7 +711,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -724,7 +724,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -838,7 +838,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.897002+00:00", + "create_time": "2024-06-18 12:45:49.756539+00:00", "table_size": "145 KB" }, "name": "date_dimension", @@ -875,7 +875,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -888,7 +888,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -901,7 +901,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=18)", + "nativeDataType": "VARCHAR(18)", "recursive": false, "isPartOfKey": false }, @@ -914,7 +914,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(9)", "recursive": false, "isPartOfKey": false }, @@ -927,7 +927,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -940,7 +940,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -953,7 +953,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -966,7 +966,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -979,7 +979,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -992,7 +992,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1005,7 +1005,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1018,7 +1018,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(9)", "recursive": false, "isPartOfKey": false }, @@ -1031,7 +1031,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1044,7 +1044,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -1057,7 +1057,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1070,7 +1070,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -1083,7 +1083,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1096,7 +1096,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1109,7 +1109,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=10)", + "nativeDataType": "VARCHAR(10)", "recursive": false, "isPartOfKey": false }, @@ -1122,7 +1122,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -1135,7 +1135,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false } @@ -1249,7 +1249,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.903227+00:00", + "create_time": "2024-06-18 12:45:49.761468+00:00", "table_size": "327 KB" }, "name": "employee_dimension", @@ -1286,7 +1286,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1299,7 +1299,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -1312,7 +1312,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -1325,7 +1325,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -1338,7 +1338,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -1351,7 +1351,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -1364,7 +1364,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1377,7 +1377,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -1390,7 +1390,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -1403,7 +1403,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -1416,7 +1416,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -1429,7 +1429,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -1442,7 +1442,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -1455,7 +1455,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1468,7 +1468,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1481,7 +1481,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1494,7 +1494,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -1507,7 +1507,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1621,7 +1621,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.912348+00:00", + "create_time": "2024-06-18 12:45:49.768272+00:00", "table_size": "2567 KB" }, "name": "inventory_fact", @@ -1658,7 +1658,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1671,7 +1671,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1684,7 +1684,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1697,7 +1697,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1710,7 +1710,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1723,7 +1723,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -1837,7 +1837,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:31.194163+00:00", + "create_time": "2024-06-18 12:46:11.881911+00:00", "table_size": "0 KB" }, "name": "phrases", @@ -1874,7 +1874,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false } @@ -1988,7 +1988,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.890782+00:00", + "create_time": "2024-06-18 12:45:49.751522+00:00", "table_size": "19 KB" }, "name": "product_dimension", @@ -2025,7 +2025,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -2038,7 +2038,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2051,7 +2051,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -2064,7 +2064,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2077,7 +2077,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2090,7 +2090,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2103,7 +2103,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2116,7 +2116,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2129,7 +2129,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2142,7 +2142,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2155,7 +2155,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2168,7 +2168,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2181,7 +2181,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2194,7 +2194,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2207,7 +2207,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2220,7 +2220,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2233,7 +2233,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2246,7 +2246,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2259,7 +2259,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2272,7 +2272,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2285,7 +2285,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -2399,7 +2399,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.893891+00:00", + "create_time": "2024-06-18 12:45:49.754039+00:00", "table_size": "3 KB" }, "name": "promotion_dimension", @@ -2436,7 +2436,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -2449,7 +2449,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -2462,7 +2462,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2475,7 +2475,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2488,7 +2488,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2501,7 +2501,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2514,7 +2514,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2527,7 +2527,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2540,7 +2540,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -2553,7 +2553,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2566,7 +2566,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -2579,7 +2579,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -2693,7 +2693,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:31.046829+00:00", + "create_time": "2024-06-18 12:46:11.751917+00:00", "table_size": "0 KB" }, "name": "readings", @@ -2730,7 +2730,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2743,7 +2743,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false }, @@ -2756,7 +2756,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -2870,7 +2870,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.906471+00:00", + "create_time": "2024-06-18 12:45:49.763788+00:00", "table_size": "1 KB" }, "name": "shipping_dimension", @@ -2907,7 +2907,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -2920,7 +2920,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=30)", + "nativeDataType": "CHAR(30)", "recursive": false, "isPartOfKey": false }, @@ -2933,7 +2933,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=10)", + "nativeDataType": "CHAR(10)", "recursive": false, "isPartOfKey": false }, @@ -2946,7 +2946,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "CHAR(20)", "recursive": false, "isPartOfKey": false } @@ -3060,7 +3060,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.900185+00:00", + "create_time": "2024-06-18 12:45:49.759103+00:00", "table_size": "1 KB" }, "name": "vendor_dimension", @@ -3097,7 +3097,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -3110,7 +3110,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -3123,7 +3123,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -3136,7 +3136,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -3149,7 +3149,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -3162,7 +3162,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -3175,7 +3175,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -3188,7 +3188,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -3302,7 +3302,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:30.233405+00:00", + "create_time": "2024-06-18 12:46:11.212820+00:00", "table_size": "0 KB" }, "name": "vmart_load_success", @@ -3339,7 +3339,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -3453,7 +3453,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.909432+00:00", + "create_time": "2024-06-18 12:45:49.766036+00:00", "table_size": "2 KB" }, "name": "warehouse_dimension", @@ -3490,7 +3490,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -3503,7 +3503,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=20)", + "nativeDataType": "VARCHAR(20)", "recursive": false, "isPartOfKey": false }, @@ -3516,7 +3516,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -3529,7 +3529,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=60)", + "nativeDataType": "VARCHAR(60)", "recursive": false, "isPartOfKey": false }, @@ -3542,7 +3542,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -3555,7 +3555,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false } @@ -3669,7 +3669,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:31.075640+00:00", + "create_time": "2024-06-18 12:46:11.778456+00:00", "table_size": "0 KB", "view_definition": "SELECT sum(customer_dimension.annual_income) AS SUM, customer_dimension.customer_state FROM public.customer_dimension WHERE (customer_dimension.customer_key IN (SELECT store_sales_fact.customer_key FROM store.store_sales_fact)) GROUP BY customer_dimension.customer_state ORDER BY customer_dimension.customer_state", "is_view": "True" @@ -3708,7 +3708,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -3721,7 +3721,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false } @@ -3962,7 +3962,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.918904+00:00", + "create_time": "2024-06-18 12:45:49.773986+00:00", "table_size": "2 KB" }, "name": "store_dimension", @@ -3999,7 +3999,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -4012,7 +4012,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -4025,7 +4025,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4038,7 +4038,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -4051,7 +4051,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -4064,7 +4064,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -4077,7 +4077,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -4090,7 +4090,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -4103,7 +4103,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -4116,7 +4116,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -4129,7 +4129,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4142,7 +4142,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4155,7 +4155,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4168,7 +4168,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4181,7 +4181,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4194,7 +4194,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4207,7 +4207,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4220,7 +4220,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -4334,7 +4334,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.929154+00:00", + "create_time": "2024-06-18 12:45:49.781744+00:00", "table_size": "8646 KB" }, "name": "store_orders_fact", @@ -4371,7 +4371,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4384,7 +4384,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4397,7 +4397,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4410,7 +4410,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4423,7 +4423,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4436,7 +4436,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4449,7 +4449,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4462,7 +4462,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4475,7 +4475,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4488,7 +4488,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4501,7 +4501,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4514,7 +4514,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4527,7 +4527,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -4540,7 +4540,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4553,7 +4553,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4566,7 +4566,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4579,7 +4579,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4592,7 +4592,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4605,7 +4605,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -4719,7 +4719,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.922050+00:00", + "create_time": "2024-06-18 12:45:49.776427+00:00", "table_size": "225096 KB" }, "name": "store_sales_fact", @@ -4756,7 +4756,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4769,7 +4769,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4782,7 +4782,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4795,7 +4795,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4808,7 +4808,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4821,7 +4821,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4834,7 +4834,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4847,7 +4847,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4860,7 +4860,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4873,7 +4873,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4886,7 +4886,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4899,7 +4899,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4912,7 +4912,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -4925,7 +4925,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIME()", + "nativeDataType": "TIME", "recursive": false, "isPartOfKey": false }, @@ -4938,7 +4938,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -4951,7 +4951,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4964,7 +4964,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false } @@ -5187,7 +5187,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.938730+00:00", + "create_time": "2024-06-18 12:45:49.789350+00:00", "table_size": "6 KB" }, "name": "call_center_dimension", @@ -5224,7 +5224,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -5237,7 +5237,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5250,7 +5250,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5263,7 +5263,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -5276,7 +5276,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -5289,7 +5289,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5302,7 +5302,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "CHAR(20)", "recursive": false, "isPartOfKey": false }, @@ -5315,7 +5315,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, "isPartOfKey": false }, @@ -5328,7 +5328,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -5341,7 +5341,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -5354,7 +5354,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -5367,7 +5367,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false } @@ -5481,7 +5481,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.935745+00:00", + "create_time": "2024-06-18 12:45:49.786989+00:00", "table_size": "9 KB" }, "name": "online_page_dimension", @@ -5518,7 +5518,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -5531,7 +5531,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5544,7 +5544,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5557,7 +5557,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5570,7 +5570,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, "isPartOfKey": false }, @@ -5583,7 +5583,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, "isPartOfKey": false } @@ -5697,7 +5697,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.941712+00:00", + "create_time": "2024-06-18 12:45:49.791761+00:00", "table_size": "182385 KB" }, "name": "online_sales_fact", @@ -5734,7 +5734,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5747,7 +5747,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5760,7 +5760,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5773,7 +5773,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5786,7 +5786,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5799,7 +5799,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5812,7 +5812,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5825,7 +5825,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5838,7 +5838,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5851,7 +5851,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5864,7 +5864,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5877,7 +5877,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5890,7 +5890,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5903,7 +5903,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5916,7 +5916,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5929,7 +5929,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5942,7 +5942,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5955,7 +5955,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -5968,7 +5968,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5981,7 +5981,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -6052,7 +6052,7 @@ "env": "PROD", "database": "vmart", "cluster_type": "Enterprise", - "cluster_size": "52 GB", + "cluster_size": "243 GB", "subcluster": " ", "communal_storage_path": "" }, @@ -6129,7 +6129,7 @@ "schema": "public", "projection_count": "12", "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, Unnest, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_kprototypes, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, arima_bfgs, arima_line_search, arima_save_model, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, chi_squared, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_arima, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_poisson_reg, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, tokenize, topk, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_predictor_importance, xgb_prune, xgb_save_model, yule_walker, ", - "udx_language": "ComplexTypesLib -- Functions for Complex Types | DelimitedExportLib -- Delimited data export package | JsonExportLib -- Json data export package | MachineLearningLib -- Machine learning package | OrcExportLib -- Orc export package | ParquetExportLib -- Parquet export package | ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | KafkaLib -- Kafka streaming load and export | PlaceLib -- Geospatial package | VoltageSecureLib -- Voltage SecureData Connector | TransformFunctions -- User-defined Python library | " + "udx_language": "ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | OrcExportLib -- Orc export package | JsonExportLib -- Json data export package | PlaceLib -- Geospatial package | ParquetExportLib -- Parquet export package | ComplexTypesLib -- Functions for Complex Types | VoltageSecureLib -- Voltage SecureData Connector | KafkaLib -- Kafka streaming load and export | MachineLearningLib -- Machine learning package | DelimitedExportLib -- Delimited data export package | TransformFunctions -- User-defined Python library | " }, "name": "public" } @@ -6304,7 +6304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6317,7 +6317,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -6330,7 +6330,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=18)", + "nativeDataType": "VARCHAR(18)", "recursive": false, "isPartOfKey": false }, @@ -6343,7 +6343,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(9)", "recursive": false, "isPartOfKey": false }, @@ -6356,7 +6356,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6369,7 +6369,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6382,7 +6382,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6395,7 +6395,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6408,7 +6408,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6421,7 +6421,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6434,7 +6434,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6447,7 +6447,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(9)", "recursive": false, "isPartOfKey": false }, @@ -6460,7 +6460,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6473,7 +6473,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -6486,7 +6486,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6499,7 +6499,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -6512,7 +6512,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6525,7 +6525,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6538,7 +6538,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=10)", + "nativeDataType": "VARCHAR(10)", "recursive": false, "isPartOfKey": false }, @@ -6551,7 +6551,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -6564,7 +6564,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false } @@ -6721,7 +6721,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6734,7 +6734,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6747,7 +6747,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -6760,7 +6760,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6773,7 +6773,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6786,7 +6786,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6799,7 +6799,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6812,7 +6812,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6825,7 +6825,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6838,7 +6838,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6851,7 +6851,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6864,7 +6864,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6877,7 +6877,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6890,7 +6890,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6903,7 +6903,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6916,7 +6916,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6929,7 +6929,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6942,7 +6942,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6955,7 +6955,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6968,7 +6968,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6981,7 +6981,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -7138,7 +7138,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7151,7 +7151,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -7164,7 +7164,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7177,7 +7177,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7190,7 +7190,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7203,7 +7203,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7216,7 +7216,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7229,7 +7229,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7242,7 +7242,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -7255,7 +7255,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7268,7 +7268,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -7281,7 +7281,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -7438,7 +7438,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7451,7 +7451,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7464,7 +7464,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7477,7 +7477,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7490,7 +7490,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -7503,7 +7503,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7516,7 +7516,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7529,7 +7529,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -7686,7 +7686,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7699,7 +7699,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -7712,7 +7712,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -7725,7 +7725,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -7738,7 +7738,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -7751,7 +7751,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7764,7 +7764,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -7777,7 +7777,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7790,7 +7790,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -7803,7 +7803,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7816,7 +7816,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7829,7 +7829,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7842,7 +7842,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7855,7 +7855,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7868,7 +7868,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7881,7 +7881,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7894,7 +7894,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7907,7 +7907,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -7920,7 +7920,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7933,7 +7933,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7946,7 +7946,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -8103,7 +8103,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8116,7 +8116,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -8129,7 +8129,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -8142,7 +8142,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -8155,7 +8155,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -8168,7 +8168,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -8181,7 +8181,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8194,7 +8194,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -8207,7 +8207,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -8220,7 +8220,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -8233,7 +8233,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -8246,7 +8246,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -8259,7 +8259,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -8272,7 +8272,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8285,7 +8285,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8298,7 +8298,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8311,7 +8311,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -8324,7 +8324,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -8481,7 +8481,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8494,7 +8494,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=20)", + "nativeDataType": "VARCHAR(20)", "recursive": false, "isPartOfKey": false }, @@ -8507,7 +8507,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -8520,7 +8520,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=60)", + "nativeDataType": "VARCHAR(60)", "recursive": false, "isPartOfKey": false }, @@ -8533,7 +8533,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -8546,7 +8546,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false } @@ -8703,7 +8703,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8716,7 +8716,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=30)", + "nativeDataType": "CHAR(30)", "recursive": false, "isPartOfKey": false }, @@ -8729,7 +8729,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=10)", + "nativeDataType": "CHAR(10)", "recursive": false, "isPartOfKey": false }, @@ -8742,7 +8742,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "CHAR(20)", "recursive": false, "isPartOfKey": false } @@ -8899,7 +8899,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8912,7 +8912,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8925,7 +8925,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8938,7 +8938,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8951,7 +8951,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8964,7 +8964,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -9121,7 +9121,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9134,7 +9134,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false }, @@ -9147,7 +9147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -9304,7 +9304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -9461,7 +9461,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false } @@ -9709,7 +9709,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9722,7 +9722,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -9735,7 +9735,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9748,7 +9748,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -9761,7 +9761,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -9774,7 +9774,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -9787,7 +9787,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -9800,7 +9800,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -9813,7 +9813,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -9826,7 +9826,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -9839,7 +9839,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9852,7 +9852,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9865,7 +9865,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -9878,7 +9878,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -9891,7 +9891,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9904,7 +9904,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9917,7 +9917,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9930,7 +9930,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -10087,7 +10087,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10100,7 +10100,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10113,7 +10113,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10126,7 +10126,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10139,7 +10139,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10152,7 +10152,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10165,7 +10165,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10178,7 +10178,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10191,7 +10191,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10204,7 +10204,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10217,7 +10217,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10230,7 +10230,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10243,7 +10243,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -10256,7 +10256,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIME()", + "nativeDataType": "TIME", "recursive": false, "isPartOfKey": false }, @@ -10269,7 +10269,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -10282,7 +10282,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10295,7 +10295,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false } @@ -10452,7 +10452,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10465,7 +10465,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10478,7 +10478,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10491,7 +10491,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10504,7 +10504,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10517,7 +10517,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10530,7 +10530,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10543,7 +10543,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10556,7 +10556,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10569,7 +10569,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10582,7 +10582,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10595,7 +10595,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10608,7 +10608,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -10621,7 +10621,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10634,7 +10634,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10647,7 +10647,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10660,7 +10660,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10673,7 +10673,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10686,7 +10686,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -10934,7 +10934,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10947,7 +10947,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10960,7 +10960,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10973,7 +10973,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10986,7 +10986,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, "isPartOfKey": false }, @@ -10999,7 +10999,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, "isPartOfKey": false } @@ -11156,7 +11156,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11169,7 +11169,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -11182,7 +11182,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -11195,7 +11195,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -11208,7 +11208,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -11221,7 +11221,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11234,7 +11234,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "CHAR(20)", "recursive": false, "isPartOfKey": false }, @@ -11247,7 +11247,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, "isPartOfKey": false }, @@ -11260,7 +11260,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -11273,7 +11273,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -11286,7 +11286,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -11299,7 +11299,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false } @@ -11456,7 +11456,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11469,7 +11469,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11482,7 +11482,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11495,7 +11495,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11508,7 +11508,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11521,7 +11521,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11534,7 +11534,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11547,7 +11547,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11560,7 +11560,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11573,7 +11573,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11586,7 +11586,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11599,7 +11599,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11612,7 +11612,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11625,7 +11625,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11638,7 +11638,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11651,7 +11651,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11664,7 +11664,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11677,7 +11677,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -11690,7 +11690,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -11703,7 +11703,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/unit/test_dbt_source.py b/metadata-ingestion/tests/unit/test_dbt_source.py index 48a6fd0f650685..01d7a4809b01b8 100644 --- a/metadata-ingestion/tests/unit/test_dbt_source.py +++ b/metadata-ingestion/tests/unit/test_dbt_source.py @@ -7,7 +7,10 @@ from datahub.emitter import mce_builder from datahub.ingestion.api.common import PipelineContext -from datahub.ingestion.source.dbt.dbt_cloud import DBTCloudConfig +from datahub.ingestion.source.dbt.dbt_cloud import ( + DBTCloudConfig, + infer_metadata_endpoint, +) from datahub.ingestion.source.dbt.dbt_core import ( DBTCoreConfig, DBTCoreSource, @@ -366,7 +369,7 @@ def test_dbt_entity_emission_configuration_helpers(): def test_dbt_cloud_config_access_url(): config_dict = { - "access_url": "https://my-dbt-cloud.dbt.com", + "access_url": "https://emea.getdbt.com", "token": "dummy_token", "account_id": "123456", "project_id": "1234567", @@ -375,8 +378,8 @@ def test_dbt_cloud_config_access_url(): "target_platform": "dummy_platform", } config = DBTCloudConfig.parse_obj(config_dict) - assert config.access_url == "https://my-dbt-cloud.dbt.com" - assert config.metadata_endpoint == "https://metadata.my-dbt-cloud.dbt.com/graphql" + assert config.access_url == "https://emea.getdbt.com" + assert config.metadata_endpoint == "https://metadata.emea.getdbt.com/graphql" def test_dbt_cloud_config_with_defined_metadata_endpoint(): @@ -398,6 +401,20 @@ def test_dbt_cloud_config_with_defined_metadata_endpoint(): ) +def test_infer_metadata_endpoint() -> None: + assert ( + infer_metadata_endpoint("https://cloud.getdbt.com") + == "https://metadata.cloud.getdbt.com/graphql" + ) + assert ( + infer_metadata_endpoint("https://prefix.us1.dbt.com") + == "https://prefix.metadata.us1.dbt.com/graphql" + ) + assert ( + infer_metadata_endpoint("http://dbt.corp.internal") + ) == "http://metadata.dbt.corp.internal/graphql" + + def test_dbt_time_parsing() -> None: time_formats = [ "2024-03-28T05:56:15.236210Z", diff --git a/metadata-ingestion/tests/unit/test_rest_sink.py b/metadata-ingestion/tests/unit/test_rest_sink.py index efa6c6678a8c75..a76f96039c2c71 100644 --- a/metadata-ingestion/tests/unit/test_rest_sink.py +++ b/metadata-ingestion/tests/unit/test_rest_sink.py @@ -1,7 +1,9 @@ import json +from datetime import datetime, timezone import pytest import requests +from freezegun import freeze_time import datahub.metadata.schema_classes as models from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -9,6 +11,7 @@ MOCK_GMS_ENDPOINT = "http://fakegmshost:8080" +FROZEN_TIME = 1618987484580 basicAuditStamp = models.AuditStampClass( time=1618987484580, actor="urn:li:corpuser:datahub", @@ -76,7 +79,7 @@ } }, "systemMetadata": { - "lastObserved": 0, + "lastObserved": FROZEN_TIME, "lastRunId": "no-run-id-provided", "properties": { "clientId": "acryl-datahub", @@ -134,7 +137,7 @@ } }, "systemMetadata": { - "lastObserved": 0, + "lastObserved": FROZEN_TIME, "lastRunId": "no-run-id-provided", "properties": { "clientId": "acryl-datahub", @@ -178,7 +181,7 @@ } }, "systemMetadata": { - "lastObserved": 0, + "lastObserved": FROZEN_TIME, "lastRunId": "no-run-id-provided", "properties": { "clientId": "acryl-datahub", @@ -263,7 +266,7 @@ "contentType": "application/json", }, "systemMetadata": { - "lastObserved": 0, + "lastObserved": FROZEN_TIME, "lastRunId": "no-run-id-provided", "properties": { "clientId": "acryl-datahub", @@ -276,6 +279,7 @@ ), ], ) +@freeze_time(datetime.fromtimestamp(FROZEN_TIME / 1000, tz=timezone.utc)) def test_datahub_rest_emitter(requests_mock, record, path, snapshot): def match_request_text(request: requests.Request) -> bool: requested_snapshot = request.json() diff --git a/metadata-ingestion/tests/unit/utilities/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/utilities/test_incremental_lineage_helper.py new file mode 100644 index 00000000000000..1db4e48fce6088 --- /dev/null +++ b/metadata-ingestion/tests/unit/utilities/test_incremental_lineage_helper.py @@ -0,0 +1,40 @@ +from typing import Optional + +from datahub.ingestion.api.incremental_lineage_helper import convert_chart_info_to_patch +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.metadata.schema_classes import ( + ChangeAuditStampsClass, + ChartInfoClass, + MetadataChangeProposalClass, +) + + +def test_convert_chart_info_to_patch(): + chart_info_class: ChartInfoClass = ChartInfoClass( + title="foo", + description="Checking patch", + inputs=[ + "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + ], + lastModified=ChangeAuditStampsClass(), + ) + + mw: Optional[MetadataWorkUnit] = convert_chart_info_to_patch( + urn="urn:li:chart:(looker,dashboard_elements.1)", + aspect=chart_info_class, + system_metadata=None, + ) + + assert mw + + assert mw.id == "urn:li:chart:(looker,dashboard_elements.1)-chartInfo" + + assert isinstance(mw.metadata, MetadataChangeProposalClass) + + assert mw.metadata.aspect + + assert ( + mw.metadata.aspect.value + == b'[{"op": "add", "path": "/title", "value": "foo"}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}, {"op": "add", "path": "/description", "value": "Checking patch"}, {"op": "add", "path": "/inputs/urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)", "value": "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)"}, {"op": "add", "path": "/inputs/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", "value": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)"}]' + ) diff --git a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py index 6c719d351c4c20..b080819cea95be 100644 --- a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py +++ b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py @@ -1,6 +1,8 @@ from typing import no_type_check +from unittest.mock import MagicMock from sqlalchemy import types +from sqlalchemy.engine.default import DefaultDialect from sqlalchemy_bigquery import STRUCT from datahub.metadata.schema_classes import ( @@ -17,8 +19,11 @@ def test_get_avro_schema_for_sqlalchemy_column(): + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() + schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=types.INTEGER() + column_name="test", column_type=types.INTEGER(), inspector=inspector_magic_mock ) assert len(schema_fields) == 1 assert schema_fields[0].fieldPath == "[version=2.0].[type=int].test" @@ -27,7 +32,10 @@ def test_get_avro_schema_for_sqlalchemy_column(): assert schema_fields[0].nullable is True schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=types.String(), nullable=False + column_name="test", + column_type=types.String(), + nullable=False, + inspector=inspector_magic_mock, ) assert len(schema_fields) == 1 assert schema_fields[0].fieldPath == "[version=2.0].[type=string].test" @@ -37,8 +45,13 @@ def test_get_avro_schema_for_sqlalchemy_column(): def test_get_avro_schema_for_sqlalchemy_array_column(): + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() + schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=types.ARRAY(types.FLOAT()) + column_name="test", + column_type=types.ARRAY(types.FLOAT()), + inspector=inspector_magic_mock, ) assert len(schema_fields) == 1 assert ( @@ -50,8 +63,13 @@ def test_get_avro_schema_for_sqlalchemy_array_column(): def test_get_avro_schema_for_sqlalchemy_map_column(): + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() + schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=MapType(types.String(), types.BOOLEAN()) + column_name="test", + column_type=MapType(types.String(), types.BOOLEAN()), + inspector=inspector_magic_mock, ) assert len(schema_fields) == 1 assert ( @@ -65,9 +83,13 @@ def test_get_avro_schema_for_sqlalchemy_map_column(): def test_get_avro_schema_for_sqlalchemy_struct_column() -> None: + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=STRUCT(("test", types.INTEGER())) + column_name="test", + column_type=STRUCT(("test", types.INTEGER())), + inspector=inspector_magic_mock, ) assert len(schema_fields) == 2 assert ( @@ -86,7 +108,12 @@ def test_get_avro_schema_for_sqlalchemy_struct_column() -> None: @no_type_check def test_get_avro_schema_for_sqlalchemy_unknown_column(): - schema_fields = get_schema_fields_for_sqlalchemy_column("invalid", "test") + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() + + schema_fields = get_schema_fields_for_sqlalchemy_column( + "invalid", "test", inspector=inspector_magic_mock + ) assert len(schema_fields) == 1 assert schema_fields[0].type.type == NullTypeClass() assert schema_fields[0].fieldPath == "[version=2.0].[type=null]" diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java index a9340d18749ade..d00dc09669045f 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java @@ -160,7 +160,7 @@ private static MetadataWriteResponse mapResponse(RecordMetadata metadata, Except return builder.build(); } - public Properties getKafkaConfgiProperties() { + public Properties getKafkaConfigProperties() { return kafkaConfigProperties; } } diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index ff29cb5fff47d2..17d9cb8cd14fee 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -130,6 +130,7 @@ test { // override, testng controlling parallelization // increasing >1 will merely run all tests extra times maxParallelForks = 1 + environment "ELASTIC_ID_HASH_ALGO", "MD5" } useTestNG() { suites 'src/test/resources/testng.xml' diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityApiUtils.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityApiUtils.java index 656534e24f551f..fa6d9cae45a6a4 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityApiUtils.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityApiUtils.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.entity; -import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; import com.datahub.util.RecordUtils; import com.linkedin.common.urn.Urn; @@ -37,10 +37,7 @@ public static RecordTemplate buildKeyAspect( public static SystemMetadata parseSystemMetadata(String jsonSystemMetadata) { if (jsonSystemMetadata == null || jsonSystemMetadata.equals("")) { - SystemMetadata response = new SystemMetadata(); - response.setRunId(DEFAULT_RUN_ID); - response.setLastObserved(0); - return response; + return createDefaultSystemMetadata(); } return RecordUtils.toRecordTemplate(SystemMetadata.class, jsonSystemMetadata); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 8c83e7f469fe31..3e640365f3fd2a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -2,7 +2,6 @@ import static com.linkedin.metadata.Constants.APP_SOURCE; import static com.linkedin.metadata.Constants.ASPECT_LATEST_VERSION; -import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID; import static com.linkedin.metadata.Constants.FORCE_INDEXING_KEY; import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; import static com.linkedin.metadata.Constants.SYSTEM_ACTOR; @@ -10,6 +9,7 @@ import static com.linkedin.metadata.utils.PegasusUtils.constructMCL; import static com.linkedin.metadata.utils.PegasusUtils.getDataTemplateClassFromSchema; import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; import static com.linkedin.metadata.utils.metrics.ExceptionUtils.collectMetrics; import com.codahale.metrics.Timer; @@ -1835,10 +1835,7 @@ public void ingestEntities( @Override public SystemMetadata ingestEntity( @Nonnull OperationContext opContext, Entity entity, AuditStamp auditStamp) { - SystemMetadata generatedSystemMetadata = new SystemMetadata(); - generatedSystemMetadata.setRunId(DEFAULT_RUN_ID); - generatedSystemMetadata.setLastObserved(System.currentTimeMillis()); - + SystemMetadata generatedSystemMetadata = createDefaultSystemMetadata(); ingestEntity(opContext, entity, auditStamp, generatedSystemMetadata); return generatedSystemMetadata; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java index 57002a3bfc59d5..cdfc4e985293f6 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java @@ -86,10 +86,10 @@ private String toDocument(SystemMetadata systemMetadata, String urn, String aspe private String toDocId(@Nonnull final String urn, @Nonnull final String aspect) { String rawDocId = urn + DOC_DELIMETER + aspect; - + String hashAlgo = System.getenv("ELASTIC_ID_HASH_ALGO"); try { byte[] bytesOfRawDocID = rawDocId.getBytes(StandardCharsets.UTF_8); - MessageDigest md = MessageDigest.getInstance("MD5"); + MessageDigest md = MessageDigest.getInstance(hashAlgo); byte[] thedigest = md.digest(bytesOfRawDocID); return Base64.getEncoder().encodeToString(thedigest); } catch (NoSuchAlgorithmException e) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/transformer/TimeseriesAspectTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/transformer/TimeseriesAspectTransformer.java index c0f66acaaca5af..cf0a3f1466d254 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/transformer/TimeseriesAspectTransformer.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/transformer/TimeseriesAspectTransformer.java @@ -257,7 +257,9 @@ private static Pair getTimeseriesFieldCollectionDocument( finalDocument); } - private static String getDocId(@Nonnull JsonNode document, String collectionId) { + private static String getDocId(@Nonnull JsonNode document, String collectionId) + throws IllegalArgumentException { + String hashAlgo = System.getenv("ELASTIC_ID_HASH_ALGO"); String docId = document.get(MappingsBuilder.TIMESTAMP_MILLIS_FIELD).toString(); JsonNode eventGranularity = document.get(MappingsBuilder.EVENT_GRANULARITY); if (eventGranularity != null) { @@ -276,6 +278,11 @@ private static String getDocId(@Nonnull JsonNode document, String collectionId) docId += partitionSpec.toString(); } - return DigestUtils.md5Hex(docId); + if (hashAlgo.equalsIgnoreCase("SHA-256")) { + return DigestUtils.sha256Hex(docId); + } else if (hashAlgo.equalsIgnoreCase("MD5")) { + return DigestUtils.md5Hex(docId); + } + throw new IllegalArgumentException("Hash function not handled !"); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java index b9a1817f476fba..346a1eef845923 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java @@ -1,5 +1,7 @@ package com.linkedin.metadata; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; + import com.linkedin.chart.ChartInfo; import com.linkedin.common.AuditStamp; import com.linkedin.common.ChangeAuditStamps; @@ -48,11 +50,10 @@ public static SystemMetadata createSystemMetadata( @Nonnull String runId, @Nonnull String lastRunId, @Nullable String version) { - SystemMetadata metadata = new SystemMetadata(); - metadata.setLastObserved(lastObserved); - metadata.setRunId(runId); + SystemMetadata metadata = createDefaultSystemMetadata(runId); metadata.setLastRunId(lastRunId); metadata.setVersion(version, SetMode.IGNORE_NULL); + metadata.setLastObserved(lastObserved); return metadata; } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index acdbd7855f7b0b..53f5ebfe59728e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -602,8 +602,9 @@ public void testReingestLineageAspect() throws Exception { GenericAspect aspect = GenericRecordUtils.serializeAspect(pairToIngest.get(0).getSecond()); + SystemMetadata initialSystemMetadata = AspectGenerationUtils.createSystemMetadata(1); initialChangeLog.setAspect(aspect); - initialChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1)); + initialChangeLog.setSystemMetadata(initialSystemMetadata); initialChangeLog.setEntityKeyAspect( GenericRecordUtils.serializeAspect( EntityKeyUtils.convertUrnToEntityKey( @@ -620,7 +621,7 @@ public void testReingestLineageAspect() throws Exception { restateChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1)); restateChangeLog.setPreviousAspectValue(aspect); restateChangeLog.setPreviousSystemMetadata( - simulatePullFromDB(AspectGenerationUtils.createSystemMetadata(1), SystemMetadata.class)); + simulatePullFromDB(initialSystemMetadata, SystemMetadata.class)); restateChangeLog.setEntityKeyAspect( GenericRecordUtils.serializeAspect( EntityKeyUtils.convertUrnToEntityKey( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java index 1aebc48153bbe5..b430313f5904b3 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java @@ -3,6 +3,7 @@ import static com.linkedin.metadata.search.utils.QueryUtils.EMPTY_FILTER; import static com.linkedin.metadata.search.utils.QueryUtils.newFilter; import static com.linkedin.metadata.search.utils.QueryUtils.newRelationshipFilter; +import static io.datahubproject.test.search.SearchTestUtils.getGraphQueryConfiguration; import static org.testng.Assert.*; import com.google.common.collect.ImmutableList; @@ -272,6 +273,8 @@ public int compare(RelatedEntity left, RelatedEntity right) { /** Any source and destination type value. */ protected static @Nullable List anyType = null; + protected final GraphQueryConfiguration _graphQueryConfiguration = getGraphQueryConfiguration(); + /** Timeout used to test concurrent ops in doTestConcurrentOp. */ protected Duration getTestConcurrentOpTimeout() { return Duration.ofMinutes(1); @@ -378,8 +381,7 @@ protected GraphService getPopulatedGraphService() throws Exception { } protected GraphService getLineagePopulatedGraphService() throws Exception { - return getLineagePopulatedGraphService( - GraphQueryConfiguration.testDefaults.isEnableMultiPathSearch()); + return getLineagePopulatedGraphService(_graphQueryConfiguration.isEnableMultiPathSearch()); } protected GraphService getLineagePopulatedGraphService(boolean multiPathSearch) throws Exception { @@ -1896,15 +1898,24 @@ public void testConcurrentAddEdge() throws Exception { allRelationships, outgoingRelationships, 0, - nodes * relationshipTypes * 2); + edges.size()); - Set expectedRelatedEntities = - edges.stream() - .map( - edge -> - new RelatedEntity(edge.getRelationshipType(), edge.getDestination().toString())) - .collect(Collectors.toSet()); - assertEquals(new HashSet<>(relatedEntities.entities), expectedRelatedEntities); + Set expectedRelatedEntities = convertEdgesToRelatedEntities(edges); + assertEquals( + deduplicateRelatedEntitiesByRelationshipTypeAndDestination(relatedEntities), + expectedRelatedEntities); + } + + protected Set convertEdgesToRelatedEntities(List edges) { + return edges.stream() + .map( + edge -> new RelatedEntity(edge.getRelationshipType(), edge.getDestination().toString())) + .collect(Collectors.toSet()); + } + + protected Set deduplicateRelatedEntitiesByRelationshipTypeAndDestination( + RelatedEntitiesResult relatedEntitiesResult) { + return Set.copyOf(relatedEntitiesResult.getEntities()); } @Test @@ -1933,8 +1944,10 @@ public void testConcurrentRemoveEdgesFromNode() throws Exception { allRelationships, outgoingRelationships, 0, - nodes * relationshipTypes * 2); - assertEquals(relatedEntities.entities.size(), nodes * relationshipTypes); + edges.size()); + assertEquals( + deduplicateRelatedEntitiesByRelationshipTypeAndDestination(relatedEntities).size(), + nodes * relationshipTypes); // delete all edges concurrently Stream operations = @@ -1992,8 +2005,10 @@ public void testConcurrentRemoveNodes() throws Exception { allRelationships, outgoingRelationships, 0, - nodes * relationshipTypes * 2); - assertEquals(relatedEntities.entities.size(), nodes * relationshipTypes); + edges.size()); + assertEquals( + deduplicateRelatedEntitiesByRelationshipTypeAndDestination(relatedEntities).size(), + nodes * relationshipTypes); // remove all nodes concurrently // nodes will be removed multiple times @@ -2138,30 +2153,20 @@ public void testHighlyConnectedGraphWalk() throws Exception { doTestConcurrentOp(operations); syncAfterWrite(); - Set expectedRelatedEntities = - edges.stream() - .map( - edge -> - new RelatedEntity(edge.getRelationshipType(), edge.getDestination().toString())) - .collect(Collectors.toSet()); - RelatedEntitiesResult relatedEntities = null; - for (int i = 0; i < 3; i++) { - relatedEntities = - service.findRelatedEntities( - null, - EMPTY_FILTER, - null, - EMPTY_FILTER, - allRelationships, - outgoingRelationships, - 0, - 400); - if (!new HashSet<>(relatedEntities.getEntities()).equals(expectedRelatedEntities)) { - // Sleep up to 6 seconds in case Elastic needs to catch up - Thread.sleep(2000); - } - } - assertEquals(new HashSet<>(relatedEntities.getEntities()), expectedRelatedEntities); + Set expectedRelatedEntities = convertEdgesToRelatedEntities(edges); + RelatedEntitiesResult relatedEntities = + service.findRelatedEntities( + null, + EMPTY_FILTER, + null, + EMPTY_FILTER, + allRelationships, + outgoingRelationships, + 0, + edges.size()); + assertEquals( + deduplicateRelatedEntitiesByRelationshipTypeAndDestination(relatedEntities), + expectedRelatedEntities); Urn root = dataset1Urn; EntityLineageResult lineageResult = diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java index 08c19bf8f52887..7513feb30d496b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java @@ -169,20 +169,6 @@ public void testConcurrentAddEdge() { "Neo4jGraphService does not manage to add all edges added concurrently"); } - @Test - @Override - public void testConcurrentRemoveEdgesFromNode() { - // https://github.com/datahub-project/datahub/issues/3118 - throw new SkipException("Neo4jGraphService produces duplicates"); - } - - @Test - @Override - public void testConcurrentRemoveNodes() { - // https://github.com/datahub-project/datahub/issues/3118 - throw new SkipException("Neo4jGraphService produces duplicates"); - } - @Test public void testRemoveEdge() throws Exception { DatasetUrn datasetUrn = diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java index b4ad5ce61d8f4e..06f1369ff0670c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java @@ -12,7 +12,6 @@ import com.linkedin.data.template.SetMode; import com.linkedin.metadata.aspect.models.graph.Edge; import com.linkedin.metadata.aspect.models.graph.RelatedEntity; -import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.graph.EntityLineageResult; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.GraphServiceTestBase; @@ -41,6 +40,8 @@ import java.util.Comparator; import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.junit.Assert; @@ -64,19 +65,18 @@ public abstract class SearchGraphServiceTestBase extends GraphServiceTestBase { private final IndexConvention _indexConvention = IndexConventionImpl.NO_PREFIX; private final String _indexName = _indexConvention.getIndexName(INDEX_NAME); private ElasticSearchGraphService _client; - private boolean _enableMultiPathSearch = - GraphQueryConfiguration.testDefaults.isEnableMultiPathSearch(); private static final String TAG_RELATIONSHIP = "SchemaFieldTaggedWith"; @BeforeClass public void setup() { - _client = buildService(_enableMultiPathSearch); + _client = buildService(_graphQueryConfiguration.isEnableMultiPathSearch()); _client.reindexAll(Collections.emptySet()); } @BeforeMethod public void wipe() throws Exception { + syncAfterWrite(); _client.clear(); syncAfterWrite(); } @@ -97,14 +97,10 @@ private ElasticSearchGraphService buildService(boolean enableMultiPathSearch) { } catch (EntityRegistryException e) { throw new RuntimeException(e); } - GraphQueryConfiguration configuration = GraphQueryConfiguration.testDefaults; - configuration.setEnableMultiPathSearch(enableMultiPathSearch); + _graphQueryConfiguration.setEnableMultiPathSearch(enableMultiPathSearch); ESGraphQueryDAO readDAO = new ESGraphQueryDAO( - getSearchClient(), - lineageRegistry, - _indexConvention, - GraphQueryConfiguration.testDefaults); + getSearchClient(), lineageRegistry, _indexConvention, _graphQueryConfiguration); ESGraphWriteDAO writeDAO = new ESGraphWriteDAO(_indexConvention, getBulkProcessor(), 1); return new ElasticSearchGraphService( lineageRegistry, @@ -118,8 +114,7 @@ private ElasticSearchGraphService buildService(boolean enableMultiPathSearch) { @Override @Nonnull protected GraphService getGraphService(boolean enableMultiPathSearch) { - if (enableMultiPathSearch != _enableMultiPathSearch) { - _enableMultiPathSearch = enableMultiPathSearch; + if (enableMultiPathSearch != _graphQueryConfiguration.isEnableMultiPathSearch()) { _client = buildService(enableMultiPathSearch); _client.reindexAll(Collections.emptySet()); } @@ -129,7 +124,7 @@ protected GraphService getGraphService(boolean enableMultiPathSearch) { @Override @Nonnull protected GraphService getGraphService() { - return getGraphService(GraphQueryConfiguration.testDefaults.isEnableMultiPathSearch()); + return getGraphService(_graphQueryConfiguration.isEnableMultiPathSearch()); } @Override @@ -305,26 +300,15 @@ public void testRemoveEdge() throws Exception { assertEquals(result.getTotal(), 0); } - @Test - @Override - public void testConcurrentAddEdge() { - // https://github.com/datahub-project/datahub/issues/3124 - throw new SkipException( - "This test is flaky for ElasticSearchGraphService, ~5% of the runs fail on a race condition"); - } - - @Test - @Override - public void testConcurrentRemoveEdgesFromNode() { - // https://github.com/datahub-project/datahub/issues/3118 - throw new SkipException("ElasticSearchGraphService produces duplicates"); - } - - @Test - @Override - public void testConcurrentRemoveNodes() { - // https://github.com/datahub-project/datahub/issues/3118 - throw new SkipException("ElasticSearchGraphService produces duplicates"); + // ElasticSearchGraphService produces duplicates + // https://github.com/datahub-project/datahub/issues/3118 + protected Set deduplicateRelatedEntitiesByRelationshipTypeAndDestination( + RelatedEntitiesResult relatedEntitiesResult) { + return relatedEntitiesResult.getEntities().stream() + .map( + relatedEntity -> + new RelatedEntity(relatedEntity.getRelationshipType(), relatedEntity.getUrn())) + .collect(Collectors.toSet()); } @Test diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java index 3dbbfb2cebc3f3..a9d84ae1f3aea1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java @@ -162,6 +162,7 @@ private void resetService(boolean withCache, boolean withLightingCache) { @BeforeMethod public void wipe() throws Exception { + syncAfterWrite(getBulkProcessor()); elasticSearchService.clear(operationContext); clearCache(false); syncAfterWrite(getBulkProcessor()); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java index a610cf95f827ae..445b71b2eaff62 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java @@ -108,6 +108,7 @@ private void resetSearchService() { @BeforeMethod public void wipe() throws Exception { + syncAfterWrite(getBulkProcessor()); elasticSearchService.clear(operationContext); syncAfterWrite(getBulkProcessor()); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java index 58574025aeeac3..ab5e90f77c21aa 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java @@ -70,7 +70,9 @@ public void setup() { @BeforeMethod public void wipe() throws Exception { + syncAfterWrite(getBulkProcessor()); elasticSearchService.clear(opContext); + syncAfterWrite(getBulkProcessor()); } @Nonnull diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java index 92ca4c5ed8a05e..f639e5c5fd3937 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java @@ -39,9 +39,9 @@ public abstract class IndexBuilderTestBase extends AbstractTestNGSpringContextTe @Nonnull protected abstract RestHighLevelClient getSearchClient(); - private static IndicesClient _indexClient; + private IndicesClient _indexClient; private static final String TEST_INDEX_NAME = "esindex_builder_test"; - private static ESIndexBuilder testDefaultBuilder; + private ESIndexBuilder testDefaultBuilder; @BeforeClass public void setup() { @@ -63,7 +63,7 @@ public void setup() { } @BeforeMethod - public static void wipe() throws Exception { + public void wipe() throws Exception { try { _indexClient .getAlias(new GetAliasesRequest(TEST_INDEX_NAME), RequestOptions.DEFAULT) @@ -86,7 +86,7 @@ public static void wipe() throws Exception { } } - public static GetIndexResponse getTestIndex() throws IOException { + public GetIndexResponse getTestIndex() throws IOException { return _indexClient.get( new GetIndexRequest(TEST_INDEX_NAME).includeDefaults(true), RequestOptions.DEFAULT); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java index 7067dd3a6763e7..d843191bed7413 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java @@ -44,7 +44,9 @@ public void setup() { @BeforeMethod public void wipe() throws Exception { + syncAfterWrite(getBulkProcessor()); _client.clear(); + syncAfterWrite(getBulkProcessor()); } @Nonnull diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java index b44f01d90dae40..10c6f09cb8f8d6 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java @@ -1291,7 +1291,7 @@ public void testCountByFilter() { @Test( groups = {"testCountAfterDelete"}, dependsOnGroups = {"deleteAspectValues1"}) - public void testCountByFilterAfterDelete() throws InterruptedException { + public void testCountByFilterAfterDelete() throws Exception { syncAfterWrite(getBulkProcessor()); // Test with filter Criterion hasUrnCriterion = diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java index 4cd818db34bf4b..e783c011de6d0e 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java @@ -1,6 +1,7 @@ package io.datahubproject.test.fixtures.search; import static com.linkedin.metadata.Constants.*; +import static io.datahubproject.test.search.SearchTestUtils.getGraphQueryConfiguration; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.client.JavaEntityClient; @@ -8,7 +9,6 @@ import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; -import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.entity.EntityServiceImpl; @@ -172,10 +172,7 @@ protected ElasticSearchGraphService graphService( indexConvention, new ESGraphWriteDAO(indexConvention, bulkProcessor, 1), new ESGraphQueryDAO( - searchClient, - lineageRegistry, - indexConvention, - GraphQueryConfiguration.testDefaults), + searchClient, lineageRegistry, indexConvention, getGraphQueryConfiguration()), indexBuilder); graphService.reindexAll(Collections.emptySet()); return graphService; diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/BulkProcessorProxyListener.java b/metadata-io/src/test/java/io/datahubproject/test/search/BulkProcessorProxyListener.java new file mode 100644 index 00000000000000..a409a1e97ad909 --- /dev/null +++ b/metadata-io/src/test/java/io/datahubproject/test/search/BulkProcessorProxyListener.java @@ -0,0 +1,44 @@ +package io.datahubproject.test.search; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.opensearch.action.bulk.BulkProcessor; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; + +public class BulkProcessorProxyListener implements BulkProcessor.Listener { + private final BulkProcessor.Listener listener; + private final AtomicInteger unsentItemsCounter = new AtomicInteger(); + + public BulkProcessorProxyListener(BulkProcessor.Listener listener) { + this.listener = listener; + } + + @Override + public void beforeBulk(long l, BulkRequest bulkRequest) { + unsentItemsCounter.addAndGet(bulkRequest.numberOfActions()); + listener.beforeBulk(l, bulkRequest); + } + + @Override + public void afterBulk(long l, BulkRequest bulkRequest, BulkResponse bulkResponse) { + unsentItemsCounter.addAndGet(-bulkResponse.getItems().length); + listener.afterBulk(l, bulkRequest, bulkResponse); + } + + @Override + public void afterBulk(long l, BulkRequest bulkRequest, Throwable throwable) { + listener.afterBulk(l, bulkRequest, throwable); + } + + void waitForBulkProcessed() throws InterruptedException { + for (int i = 0; i < 6000; i++) { + if (unsentItemsCounter.get() == 0) { + break; + } + TimeUnit.MILLISECONDS.sleep(5); + } + // reset the counter just in case + unsentItemsCounter.set(0); + } +} diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/BulkProcessorTestUtils.java b/metadata-io/src/test/java/io/datahubproject/test/search/BulkProcessorTestUtils.java new file mode 100644 index 00000000000000..416a5d40bb0e3f --- /dev/null +++ b/metadata-io/src/test/java/io/datahubproject/test/search/BulkProcessorTestUtils.java @@ -0,0 +1,67 @@ +package io.datahubproject.test.search; + +import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksRequest; +import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.action.bulk.BulkProcessor; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.common.unit.TimeValue; +import org.springframework.test.util.ReflectionTestUtils; + +public class BulkProcessorTestUtils { + private BulkProcessorTestUtils() {} + + public static void syncAfterWrite(ESBulkProcessor bulkProcessor) + throws InterruptedException, IOException { + bulkProcessor.flush(); + final RestHighLevelClient searchClient = getRestHighLevelClient(bulkProcessor); + // if the bulks are big it takes time for Elastic/OpenSearch to process these bulk requests + getBulkProcessorListener(bulkProcessor).waitForBulkProcessed(); + waitForCompletion(searchClient); + // some tasks might have refresh = false, so we need to refresh manually + searchClient.indices().refresh(new RefreshRequest(), RequestOptions.DEFAULT); + waitForCompletion(searchClient); + } + + private static void waitForCompletion(RestHighLevelClient searchClient) + throws IOException, InterruptedException { + while (!searchClient + .tasks() + .list( + new ListTasksRequest() + .setActions("indices:*,*/put,*/update") + .setWaitForCompletion(true) + .setTimeout(TimeValue.timeValueSeconds(30)), + RequestOptions.DEFAULT) + .getTasks() + .isEmpty()) { + // Mostly this is not reached, but in some rare cases it might + TimeUnit.MILLISECONDS.sleep(5); + } + } + + private static RestHighLevelClient getRestHighLevelClient(ESBulkProcessor esBulkProcessor) { + return (RestHighLevelClient) ReflectionTestUtils.getField(esBulkProcessor, "searchClient"); + } + + private static BulkProcessorProxyListener getBulkProcessorListener( + ESBulkProcessor esBulkProcessor) { + var bulkProcessor = ReflectionTestUtils.getField(esBulkProcessor, "bulkProcessor"); + var bulkRequestHandler = ReflectionTestUtils.getField(bulkProcessor, "bulkRequestHandler"); + return (BulkProcessorProxyListener) + ReflectionTestUtils.getField(bulkRequestHandler, "listener"); + } + + public static void replaceBulkProcessorListener(ESBulkProcessor esBulkProcessor) { + var bulkProcessor = + (BulkProcessor) ReflectionTestUtils.getField(esBulkProcessor, "bulkProcessor"); + var bulkRequestHandler = ReflectionTestUtils.getField(bulkProcessor, "bulkRequestHandler"); + var bulkProcessorListener = + (BulkProcessor.Listener) ReflectionTestUtils.getField(bulkRequestHandler, "listener"); + ReflectionTestUtils.setField( + bulkRequestHandler, "listener", new BulkProcessorProxyListener(bulkProcessorListener)); + } +} diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java index a71c40b70f2b41..24df2afb3b7819 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java @@ -15,6 +15,7 @@ import com.linkedin.datahub.graphql.types.SearchableEntityType; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.LineageSearchResult; @@ -24,6 +25,7 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import io.datahubproject.metadata.context.OperationContext; +import java.io.IOException; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -41,9 +43,9 @@ public class SearchTestUtils { private SearchTestUtils() {} - public static void syncAfterWrite(ESBulkProcessor bulkProcessor) throws InterruptedException { - bulkProcessor.flush(); - Thread.sleep(1000); + public static void syncAfterWrite(ESBulkProcessor bulkProcessor) + throws InterruptedException, IOException { + BulkProcessorTestUtils.syncAfterWrite(bulkProcessor); } public static final List SEARCHABLE_ENTITIES; @@ -253,4 +255,16 @@ public HttpAsyncClientBuilder customizeHttpClient( } }); } + + public static GraphQueryConfiguration getGraphQueryConfiguration() { + return new GraphQueryConfiguration() { + { + setBatchSize(1000); + setTimeoutSeconds(10); + setMaxResult(10000); + setEnableMultiPathSearch(true); + setBoostViaNodes(true); + } + }; + } } diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java index ab6644ce6ff6de..66394def5f99b5 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java @@ -1,5 +1,7 @@ package io.datahubproject.test.search.config; +import static io.datahubproject.test.search.BulkProcessorTestUtils.replaceBulkProcessorListener; + import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; @@ -64,18 +66,21 @@ public RestHighLevelClient getElasticsearchClient( @Nonnull public ESBulkProcessor getBulkProcessor( @Qualifier("searchRestHighLevelClient") RestHighLevelClient searchClient) { - return ESBulkProcessor.builder(searchClient) - .async(true) - /* - * Force a refresh as part of this request. This refresh policy does not scale for high indexing or search throughput but is useful - * to present a consistent view to for indices with very low traffic. And it is wonderful for tests! - */ - .writeRequestRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .bulkRequestsLimit(10000) - .bulkFlushPeriod(REFRESH_INTERVAL_SECONDS - 1) - .retryInterval(1L) - .numRetries(1) - .build(); + ESBulkProcessor esBulkProcessor = + ESBulkProcessor.builder(searchClient) + .async(true) + /* + * Force a refresh as part of this request. This refresh policy does not scale for high indexing or search throughput but is useful + * to present a consistent view to for indices with very low traffic. And it is wonderful for tests! + */ + .writeRequestRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .bulkRequestsLimit(10000) + .bulkFlushPeriod(REFRESH_INTERVAL_SECONDS - 1) + .retryInterval(1L) + .numRetries(1) + .build(); + replaceBulkProcessorListener(esBulkProcessor); + return esBulkProcessor; } @Primary diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionSummary.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionSummary.pdl index 3984277a314171..da126fa7d83d77 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionSummary.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionSummary.pdl @@ -1,7 +1,5 @@ namespace com.linkedin.dataset -import com.linkedin.common.AuditStamp - /** * Defines how the data is partitioned */ @@ -15,10 +13,10 @@ record PartitionSummary { /** * The created time for a given partition. */ - created: optional AuditStamp + createdTime: optional long /** * The last modified / touched time for a given partition. */ - lastModified: optional AuditStamp + lastModifiedTime: optional long } \ No newline at end of file diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/services/SecretService.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/services/SecretService.java index bea03235abfb4c..48e7f80173cfde 100644 --- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/services/SecretService.java +++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/services/SecretService.java @@ -50,7 +50,7 @@ public String encrypt(String value) { } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } - Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding"); + Cipher cipher = Cipher.getInstance("AES"); cipher.init(Cipher.ENCRYPT_MODE, secretKey); return _encoder.encodeToString(cipher.doFinal(value.getBytes(StandardCharsets.UTF_8))); } catch (Exception e) { @@ -72,7 +72,7 @@ public String decrypt(String encryptedValue) { } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } - Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING"); + Cipher cipher = Cipher.getInstance("AES"); cipher.init(Cipher.DECRYPT_MODE, secretKey); return new String(cipher.doFinal(_decoder.decode(encryptedValue))); } catch (Exception e) { diff --git a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java index dd68e4d36f14e4..ee2efd2ae95365 100644 --- a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java +++ b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java @@ -92,7 +92,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha "Failed to authenticate request. Received an AuthenticationExpiredException from authenticator chain.", e); ((HttpServletResponse) response) - .sendError(HttpServletResponse.SC_UNAUTHORIZED, e.getMessage()); + .sendError(HttpServletResponse.SC_UNAUTHORIZED, "Unauthorized to perform this action."); return; } diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java index cd869a61bf3abb..7a4af8c24262ed 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java @@ -17,15 +17,4 @@ public class GraphQueryConfiguration { * to be prioritized in the case of a multiple path situation with multi-path search disabled */ private boolean boostViaNodes; - - public static GraphQueryConfiguration testDefaults; - - static { - testDefaults = new GraphQueryConfiguration(); - testDefaults.setBatchSize(1000); - testDefaults.setTimeoutSeconds(10); - testDefaults.setMaxResult(10000); - testDefaults.setEnableMultiPathSearch(true); - testDefaults.setBoostViaNodes(true); - } } diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 921c2b43dc36c6..2514060ff2d617 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -174,6 +174,7 @@ elasticsearch: opensearchUseAwsIamAuth: ${OPENSEARCH_USE_AWS_IAM_AUTH:false} region: ${AWS_REGION:#{null}} implementation: ${ELASTICSEARCH_IMPLEMENTATION:elasticsearch} # elasticsearch or opensearch, for handling divergent cases + idHashAlgo: ${ELASTIC_ID_HASH_ALGO:MD5} sslContext: # Required if useSSL is true protocol: ${ELASTICSEARCH_SSL_PROTOCOL:#{null}} secureRandomImplementation: ${ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL:#{null}} diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java index 6743effd775b1b..668892bb46b7fe 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.boot.steps; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -23,7 +24,6 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; -import com.linkedin.mxe.SystemMetadata; import io.datahubproject.metadata.context.OperationContext; import java.util.Set; import javax.annotation.Nonnull; @@ -152,8 +152,7 @@ private void ingestBrowsePathsV2( proposal.setEntityType(urn.getEntityType()); proposal.setAspectName(Constants.BROWSE_PATHS_V2_ASPECT_NAME); proposal.setChangeType(ChangeType.UPSERT); - proposal.setSystemMetadata( - new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(System.currentTimeMillis())); + proposal.setSystemMetadata(createDefaultSystemMetadata()); proposal.setAspect(GenericRecordUtils.serializeAspect(browsePathsV2)); entityService.ingestProposal(systemOperationContext, proposal, auditStamp, false); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java index 638d1a6fd0c8ac..22ce06a5984ea6 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java @@ -61,7 +61,7 @@ public void execute(@Nonnull OperationContext systemOperationContext) throws Exc long numEntities = _migrationsDao.countEntities(); int start = 0; - while (start < numEntities) { + while (start < (int) numEntities) { log.info( "Reading urns {} to {} from the aspects table to generate dataplatform instance aspects", start, diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java index 765ba2f44a54d1..89846476a9875e 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.boot.steps; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.utils.SystemMetadataUtils.createDefaultSystemMetadata; import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; @@ -17,7 +18,6 @@ import com.linkedin.metadata.search.utils.BrowsePathUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; -import com.linkedin.mxe.SystemMetadata; import io.datahubproject.metadata.context.OperationContext; import java.util.Set; import javax.annotation.Nonnull; @@ -138,8 +138,7 @@ private void migrateBrowsePath( proposal.setEntityType(urn.getEntityType()); proposal.setAspectName(Constants.BROWSE_PATHS_ASPECT_NAME); proposal.setChangeType(ChangeType.UPSERT); - proposal.setSystemMetadata( - new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(System.currentTimeMillis())); + proposal.setSystemMetadata(createDefaultSystemMetadata()); proposal.setAspect(GenericRecordUtils.serializeAspect(newPaths)); entityService.ingestProposal(opContext, proposal, auditStamp, false); } diff --git a/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java b/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java index 2f383b1956313d..f04f5f80d03a41 100644 --- a/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java +++ b/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java @@ -75,7 +75,7 @@ CompletableFuture> postGraphQL( try { bodyJson = mapper.readTree(jsonStr); } catch (JsonProcessingException e) { - log.error("Failed to parse json {}", jsonStr); + log.error("Failed to parse json ", e); return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index e79dda34256822..f2e025c8ed2b43 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -13,6 +13,7 @@ import static com.linkedin.metadata.resources.restli.RestliConstants.*; import static com.linkedin.metadata.search.utils.SearchUtils.*; import static com.linkedin.metadata.utils.PegasusUtils.*; +import static com.linkedin.metadata.utils.SystemMetadataUtils.generateSystemMetadataIfEmpty; import com.codahale.metrics.MetricRegistry; import com.datahub.authentication.Authentication; @@ -20,6 +21,7 @@ import com.datahub.authorization.AuthUtil; import com.datahub.authorization.EntitySpec; +import com.linkedin.metadata.utils.SystemMetadataUtils; import io.datahubproject.metadata.context.RequestContext; import io.datahubproject.metadata.services.RestrictedService; import com.linkedin.data.template.SetMode; @@ -257,19 +259,6 @@ public Task> batchGet( MetricRegistry.name(this.getClass(), "batchGet")); } - private SystemMetadata populateDefaultFieldsIfEmpty(@Nullable SystemMetadata systemMetadata) { - SystemMetadata result = systemMetadata; - if (result == null) { - result = new SystemMetadata(); - } - - if (result.getLastObserved() == 0) { - result.setLastObserved(System.currentTimeMillis()); - } - - return result; - } - @Action(name = ACTION_INGEST) @Nonnull @WithSpan @@ -297,7 +286,7 @@ public Task ingest( throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, e); } - SystemMetadata systemMetadata = populateDefaultFieldsIfEmpty(providedSystemMetadata); + SystemMetadata systemMetadata = generateSystemMetadataIfEmpty(providedSystemMetadata); final AuditStamp auditStamp = new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr)); @@ -358,7 +347,7 @@ public Task batchIngest( final List finalSystemMetadataList = Arrays.stream(systemMetadataList) - .map(systemMetadata -> populateDefaultFieldsIfEmpty(systemMetadata)) + .map(SystemMetadataUtils::generateSystemMetadataIfEmpty) .collect(Collectors.toList()); return RestliUtil.toTask( diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java index 01af399c9b1155..403665120c6868 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java @@ -173,7 +173,7 @@ public RollbackResponse rollbackIngestion( // Rollback timeseries aspects DeleteAspectValuesResult timeseriesRollbackResult = timeseriesAspectService.rollbackTimeseriesAspects(opContext, runId); - rowsDeletedFromEntityDeletion += timeseriesRollbackResult.getNumDocsDeleted(); + rowsDeletedFromEntityDeletion += timeseriesRollbackResult.getNumDocsDeleted().intValue(); log.info("finished deleting {} rows", deletedRows.size()); int aspectsReverted = deletedRows.size() + rowsDeletedFromEntityDeletion; diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java index c3c9cac6280ed2..aa18124c826dac 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java @@ -54,10 +54,10 @@ public static List convertToFilters( public static FilterValue createFilterValue(String value, Long facetCount, Boolean isFilteredOn) { // TODO(indy): test this - String[] aggregationTokens = value.split(AGGREGATION_SEPARATOR_CHAR); + String[] aggregations = value.split(AGGREGATION_SEPARATOR_CHAR); FilterValue result = new FilterValue().setValue(value).setFacetCount(facetCount).setFiltered(isFilteredOn); - String lastValue = aggregationTokens[aggregationTokens.length - 1]; + String lastValue = aggregations[aggregations.length - 1]; if (lastValue.startsWith(URN_PREFIX)) { try { result.setEntity(Urn.createFromString(lastValue)); diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SystemMetadataUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SystemMetadataUtils.java index 81bfcaab74ddb8..4cc09f83e5f740 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SystemMetadataUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SystemMetadataUtils.java @@ -1,6 +1,8 @@ package com.linkedin.metadata.utils; -import com.linkedin.metadata.Constants; +import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID; + +import com.linkedin.data.template.SetMode; import com.linkedin.mxe.SystemMetadata; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -11,13 +13,25 @@ public class SystemMetadataUtils { private SystemMetadataUtils() {} public static SystemMetadata createDefaultSystemMetadata() { - return new SystemMetadata() - .setRunId(Constants.DEFAULT_RUN_ID) - .setLastObserved(System.currentTimeMillis()); + return generateSystemMetadataIfEmpty(null); + } + + public static SystemMetadata createDefaultSystemMetadata(@Nullable String runId) { + return generateSystemMetadataIfEmpty( + new SystemMetadata() + .setRunId(runId, SetMode.REMOVE_IF_NULL) + .setLastObserved(System.currentTimeMillis())); } public static SystemMetadata generateSystemMetadataIfEmpty( @Nullable SystemMetadata systemMetadata) { - return systemMetadata == null ? createDefaultSystemMetadata() : systemMetadata; + SystemMetadata result = systemMetadata == null ? new SystemMetadata() : systemMetadata; + if (result.getRunId() == null) { + result.setRunId(DEFAULT_RUN_ID); + } + if (!result.hasLastObserved() || result.getLastObserved() == 0) { + result.setLastObserved(System.currentTimeMillis()); + } + return result; } } diff --git a/smoke-test/build.gradle b/smoke-test/build.gradle index 9800cf65fc4529..95f3ba8ed56d64 100644 --- a/smoke-test/build.gradle +++ b/smoke-test/build.gradle @@ -89,6 +89,7 @@ task noCypressSuite0(type: Exec, dependsOn: [installDev, ':metadata-ingestion:in environment 'DATAHUB_KAFKA_SCHEMA_REGISTRY_URL', 'http://localhost:8080/schema-registry/api/' environment 'KAFKA_BROKER_CONTAINER', 'datahub-kafka-broker-1' environment 'TEST_STRATEGY', 'no_cypress_suite0' + environment "ELASTIC_ID_HASH_ALGO", "MD5" workingDir = project.projectDir commandLine 'bash', '-c', @@ -101,6 +102,7 @@ task noCypressSuite1(type: Exec, dependsOn: [installDev, ':metadata-ingestion:in environment 'DATAHUB_KAFKA_SCHEMA_REGISTRY_URL', 'http://localhost:8080/schema-registry/api/' environment 'KAFKA_BROKER_CONTAINER', 'datahub-kafka-broker-1' environment 'TEST_STRATEGY', 'no_cypress_suite1' + environment "ELASTIC_ID_HASH_ALGO", "MD5" workingDir = project.projectDir commandLine 'bash', '-c', @@ -113,6 +115,7 @@ task cypressSuite1(type: Exec, dependsOn: [installDev, ':metadata-ingestion:inst environment 'DATAHUB_KAFKA_SCHEMA_REGISTRY_URL', 'http://localhost:8080/schema-registry/api/' environment 'KAFKA_BROKER_CONTAINER', 'datahub-kafka-broker-1' environment 'TEST_STRATEGY', 'cypress_suite1' + environment "ELASTIC_ID_HASH_ALGO", "MD5" workingDir = project.projectDir commandLine 'bash', '-c', @@ -125,6 +128,7 @@ task cypressRest(type: Exec, dependsOn: [installDev, ':metadata-ingestion:instal environment 'DATAHUB_KAFKA_SCHEMA_REGISTRY_URL', 'http://localhost:8080/schema-registry/api/' environment 'KAFKA_BROKER_CONTAINER', 'datahub-kafka-broker-1' environment 'TEST_STRATEGY', 'cypress_rest' + environment "ELASTIC_ID_HASH_ALGO", "MD5" workingDir = project.projectDir commandLine 'bash', '-c', @@ -139,6 +143,7 @@ task cypressDev(type: Exec, dependsOn: [installDev, ':metadata-ingestion:install environment 'RUN_QUICKSTART', 'false' environment 'DATAHUB_KAFKA_SCHEMA_REGISTRY_URL', 'http://localhost:8080/schema-registry/api/' environment 'KAFKA_BROKER_CONTAINER', 'datahub-kafka-broker-1' + environment "ELASTIC_ID_HASH_ALGO", "MD5" workingDir = project.projectDir commandLine 'bash', '-c', @@ -154,6 +159,7 @@ task cypressData(type: Exec, dependsOn: [installDev, ':metadata-ingestion:instal environment 'DATAHUB_KAFKA_SCHEMA_REGISTRY_URL', 'http://localhost:8080/schema-registry/api/' environment 'KAFKA_BROKER_CONTAINER', 'datahub-kafka-broker-1' environment 'RUN_UI', 'false' + environment "ELASTIC_ID_HASH_ALGO", "MD5" workingDir = project.projectDir commandLine 'bash', '-c', diff --git a/smoke-test/cypress-dev.sh b/smoke-test/cypress-dev.sh index 2b31c574d05787..59346b26069059 100755 --- a/smoke-test/cypress-dev.sh +++ b/smoke-test/cypress-dev.sh @@ -12,6 +12,7 @@ source venv/bin/activate export KAFKA_BROKER_CONTAINER="datahub-kafka-broker-1" export KAFKA_BOOTSTRAP_SERVER="broker:9092" +export ELASTIC_ID_HASH_ALGO="MD5" python -c 'from tests.cypress.integration_test import ingest_data; ingest_data()' cd tests/cypress diff --git a/smoke-test/run-quickstart.sh b/smoke-test/run-quickstart.sh index 05c321566d54a6..1923d42eb5e939 100755 --- a/smoke-test/run-quickstart.sh +++ b/smoke-test/run-quickstart.sh @@ -15,6 +15,7 @@ DATAHUB_SEARCH_TAG="${DATAHUB_SEARCH_TAG:=2.9.0}" XPACK_SECURITY_ENABLED="${XPACK_SECURITY_ENABLED:=plugins.security.disabled=true}" ELASTICSEARCH_USE_SSL="${ELASTICSEARCH_USE_SSL:=false}" USE_AWS_ELASTICSEARCH="${USE_AWS_ELASTICSEARCH:=true}" +ELASTIC_ID_HASH_ALGO="${ELASTIC_ID_HASH_ALGO:=MD5}" echo "DATAHUB_VERSION = $DATAHUB_VERSION" DATAHUB_TELEMETRY_ENABLED=false \ diff --git a/smoke-test/smoke.sh b/smoke-test/smoke.sh index fafb2076fe6990..c16865fe1e71ef 100755 --- a/smoke-test/smoke.sh +++ b/smoke-test/smoke.sh @@ -25,6 +25,7 @@ source venv/bin/activate source ./set-cypress-creds.sh export DATAHUB_GMS_URL=http://localhost:8080 +export ELASTIC_ID_HASH_ALGO="MD5" # no_cypress_suite0, no_cypress_suite1, cypress_suite1, cypress_rest if [[ -z "${TEST_STRATEGY}" ]]; then