Merge branch 'master' into docs/customer-stories-refactor

datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipAction.less

@@ -1,5 +1,3 @@
-@import '../../../../../node_modules/antd/dist/antd.less';
-
 .joinName {
     width: 385px;
     height: 24px;

datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.less

@@ -1,5 +1,3 @@
-@import '../../../../../../../node_modules/antd/dist/antd.less';
-
 .CreateERModelRelationModal {
     .ermodelrelation-name {
         padding: 8px 16px;

datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationPreview.less

@@ -1,5 +1,3 @@
-@import '../../../../../../../node_modules/antd/dist/antd.less';
-
 .ERModelRelationPreview {
     .preview-main-div {
         display: flex;

datahub-web-react/src/app/entity/shared/tabs/Dataset/Relationship/RelationshipsTab.less

@@ -1,5 +1,3 @@
-@import '../../../../../../../node_modules/antd/dist/antd.less';
-
 .RelationshipsTab {
     .add-btn-link {
         height: 56px !important;

datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.less

@@ -1,5 +1,3 @@
-@import '../../../../../../node_modules/antd/dist/antd.less';
-
 .ERModelRelationTab {
     .add-btn-link {
         padding-left: 1155px !important;

docs/cli.md

@@ -193,14 +193,14 @@ datahub init
 /Users/user/.datahubenv already exists. Overwrite? [y/N]: y
 Configure which datahub instance to connect to
 Enter your DataHub host [http://localhost:8080]: http://localhost:8080
-Enter your DataHub access token (Supports env vars via `{VAR_NAME}` syntax) []:
+Enter your DataHub access token []:
 
 # acryl example
 datahub init
 /Users/user/.datahubenv already exists. Overwrite? [y/N]: y
 Configure which datahub instance to connect to
 Enter your DataHub host [http://localhost:8080]: https://<your-instance-id>.acryl.io/gms
-Enter your DataHub access token (Supports env vars via `{VAR_NAME}` syntax) []: <token generated from https://<your-instance-id>.acryl.io/settings/tokens>
+Enter your DataHub access token []: <token generated from https://<your-instance-id>.acryl.io/settings/tokens>
 ```
 
 #### Environment variables supported

metadata-ingestion/src/datahub/entrypoints.py

@@ -149,7 +149,7 @@ def init(use_password: bool = False) -> None:
         )
     else:
         token = click.prompt(
-            "Enter your DataHub access token (Supports env vars via `{VAR_NAME}` syntax)",
+            "Enter your DataHub access token",
             type=str,
             default="",
         )

metadata-ingestion/src/datahub/ingestion/graph/client.py

@@ -171,6 +171,22 @@ def test_connection(self) -> None:
             self.server_id = _MISSING_SERVER_ID
             logger.debug(f"Failed to get server id due to {e}")
 
+    @property
+    def frontend_base_url(self) -> str:
+        """Get the public-facing base url of the frontend
+
+        This url can be used to construct links to the frontend. The url will not include a trailing slash.
+        Note: Only supported with Acryl Cloud.
+        """
+
+        if not self.server_config:
+            self.test_connection()
+
+        base_url = self.server_config.get("baseUrl")
+        if not base_url:
+            raise ValueError("baseUrl not found in server config")
+        return base_url
+
     @classmethod
     def from_emitter(cls, emitter: DatahubRestEmitter) -> "DataHubGraph":
         return cls(
@@ -812,6 +828,7 @@ def get_urns_by_filter(
         status: RemovedStatusFilter = RemovedStatusFilter.NOT_SOFT_DELETED,
         batch_size: int = 10000,
         extraFilters: Optional[List[SearchFilterRule]] = None,
+        extra_or_filters: Optional[List[Dict[str, List[SearchFilterRule]]]] = None,
     ) -> Iterable[str]:
         """Fetch all urns that match all of the given filters.
 
@@ -841,7 +858,13 @@ def get_urns_by_filter(
 
         # Env filter.
         orFilters = generate_filter(
-            platform, platform_instance, env, container, status, extraFilters
+            platform,
+            platform_instance,
+            env,
+            container,
+            status,
+            extraFilters,
+            extra_or_filters=extra_or_filters,
         )
 
         graphql_query = textwrap.dedent(
@@ -885,6 +908,137 @@ def get_urns_by_filter(
         for entity in self._scroll_across_entities(graphql_query, variables):
             yield entity["urn"]
 
+    def get_results_by_filter(
+        self,
+        *,
+        entity_types: Optional[List[str]] = None,
+        platform: Optional[str] = None,
+        platform_instance: Optional[str] = None,
+        env: Optional[str] = None,
+        query: Optional[str] = None,
+        container: Optional[str] = None,
+        status: RemovedStatusFilter = RemovedStatusFilter.NOT_SOFT_DELETED,
+        batch_size: int = 10000,
+        extra_and_filters: Optional[List[SearchFilterRule]] = None,
+        extra_or_filters: Optional[List[Dict[str, List[SearchFilterRule]]]] = None,
+        extra_source_fields: Optional[List[str]] = None,
+        skip_cache: bool = False,
+    ) -> Iterable[dict]:
+        """Fetch all results that match all of the given filters.
+
+        Note: Only supported with Acryl Cloud.
+
+        Filters are combined conjunctively. If multiple filters are specified, the results will match all of them.
+        Note that specifying a platform filter will automatically exclude all entity types that do not have a platform.
+        The same goes for the env filter.
+
+        :param entity_types: List of entity types to include. If None, all entity types will be returned.
+        :param platform: Platform to filter on. If None, all platforms will be returned.
+        :param platform_instance: Platform instance to filter on. If None, all platform instances will be returned.
+        :param env: Environment (e.g. PROD, DEV) to filter on. If None, all environments will be returned.
+        :param query: Query string to filter on. If None, all entities will be returned.
+        :param container: A container urn that entities must be within.
+            This works recursively, so it will include entities within sub-containers as well.
+            If None, all entities will be returned.
+            Note that this requires browsePathV2 aspects (added in 0.10.4+).
+        :param status: Filter on the deletion status of the entity. The default is only return non-soft-deleted entities.
+        :param extra_and_filters: Additional filters to apply. If specified, the
+            results will match all of the filters.
+        :param extra_or_filters: Additional filters to apply. If specified, the
+            results will match any of the filters.
+
+        :return: An iterable of urns that match the filters.
+        """
+
+        types = self._get_types(entity_types)
+
+        # Add the query default of * if no query is specified.
+        query = query or "*"
+
+        or_filters_final = generate_filter(
+            platform,
+            platform_instance,
+            env,
+            container,
+            status,
+            extra_and_filters,
+            extra_or_filters,
+        )
+        graphql_query = textwrap.dedent(
+            """
+            query scrollUrnsWithFilters(
+                $types: [EntityType!],
+                $query: String!,
+                $orFilters: [AndFilterInput!],
+                $batchSize: Int!,
+                $scrollId: String,
+                $skipCache: Boolean!,
+                $fetchExtraFields: [String!]) {
+
+                scrollAcrossEntities(input: {
+                    query: $query,
+                    count: $batchSize,
+                    scrollId: $scrollId,
+                    types: $types,
+                    orFilters: $orFilters,
+                    searchFlags: {
+                        skipHighlighting: true
+                        skipAggregates: true
+                        skipCache: $skipCache
+                        fetchExtraFields: $fetchExtraFields
+                    }
+                }) {
+                    nextScrollId
+                    searchResults {
+                        entity {
+                            urn
+                        }
+                        extraProperties {
+                            name
+                            value
+                        }
+                    }
+                }
+            }
+            """
+        )
+
+        variables = {
+            "types": types,
+            "query": query,
+            "orFilters": or_filters_final,
+            "batchSize": batch_size,
+            "skipCache": "true" if skip_cache else "false",
+            "fetchExtraFields": extra_source_fields,
+        }
+
+        for result in self._scroll_across_entities_results(graphql_query, variables):
+            yield result
+
+    def _scroll_across_entities_results(
+        self, graphql_query: str, variables_orig: dict
+    ) -> Iterable[dict]:
+        variables = variables_orig.copy()
+        first_iter = True
+        scroll_id: Optional[str] = None
+        while first_iter or scroll_id:
+            first_iter = False
+            variables["scrollId"] = scroll_id
+
+            response = self.execute_graphql(
+                graphql_query,
+                variables=variables,
+            )
+            data = response["scrollAcrossEntities"]
+            scroll_id = data["nextScrollId"]
+            for entry in data["searchResults"]:
+                yield entry
+
+            if scroll_id:
+                logger.debug(
+                    f"Scrolling to next scrollAcrossEntities page: {scroll_id}"
+                )
+
     def _scroll_across_entities(
         self, graphql_query: str, variables_orig: dict
     ) -> Iterable[dict]:

metadata-ingestion/src/datahub/ingestion/graph/filters.py

@@ -30,7 +30,19 @@ def generate_filter(
     container: Optional[str],
     status: RemovedStatusFilter,
     extra_filters: Optional[List[SearchFilterRule]],
+    extra_or_filters: Optional[List[SearchFilterRule]] = None,
 ) -> List[Dict[str, List[SearchFilterRule]]]:
+    """
+    Generate a search filter based on the provided parameters.
+    :param platform: The platform to filter by.
+    :param platform_instance: The platform instance to filter by.
+    :param env: The environment to filter by.
+    :param container: The container to filter by.
+    :param status: The status to filter by.
+    :param extra_filters: Extra AND filters to apply.
+    :param extra_or_filters: Extra OR filters to apply. These are combined with
+    the AND filters using an OR at the top level.
+    """
     and_filters: List[SearchFilterRule] = []
 
     # Platform filter.
@@ -66,6 +78,14 @@ def generate_filter(
             for and_filter in or_filters
         ]
 
+    # Extra OR filters are distributed across the top level and lists.
+    if extra_or_filters:
+        or_filters = [
+            {"and": and_filter["and"] + [extra_or_filter]}
+            for extra_or_filter in extra_or_filters
+            for and_filter in or_filters
+        ]
+
     return or_filters
 
 

metadata-ingestion/tests/integration/sql_server/test_sql_server.py

@@ -22,7 +22,7 @@ def mssql_runner(docker_compose_runner, pytestconfig):
         time.sleep(5)
 
         # Run the setup.sql file to populate the database.
-        command = "docker exec testsqlserver /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'test!Password' -d master -i /setup/setup.sql"
+        command = "docker exec testsqlserver /opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P 'test!Password' -d master -i /setup/setup.sql"
         ret = subprocess.run(command, shell=True, capture_output=True)
         assert ret.returncode == 0
         yield docker_services

metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java

@@ -2176,7 +2176,8 @@ public Set<Urn> exists(
               urn ->
                   // key aspect is always returned, make sure to only consider the status aspect
                   statusResult.getOrDefault(urn, List.of()).stream()
-                      .filter(aspect -> STATUS_ASPECT_NAME.equals(aspect.schema().getName()))
+                      .filter(
+                          aspect -> STATUS_ASPECT_NAME.equalsIgnoreCase(aspect.schema().getName()))
                       .noneMatch(aspect -> ((Status) aspect).isRemoved()))
           .collect(Collectors.toSet());
     }

metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/StructuredPropertiesValidator.java

@@ -356,7 +356,7 @@ private static Optional<AspectValidationException> validateType(
                 throw new RuntimeException(e);
               }
               String allowedEntityName = getValueTypeId(typeUrn);
-              if (typeValue.getEntityType().equals(allowedEntityName)) {
+              if (typeValue.getEntityType().equalsIgnoreCase(allowedEntityName)) {
                 matchedAny = true;
               }
             }

metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java

@@ -547,12 +547,12 @@ private static void addAspectSchemas(final Components components, final AspectSp
                           String $ref = schema.get$ref();
                           boolean isNameRequired = requiredNames.contains(name);
                           if ($ref != null && !isNameRequired) {
-                            // A non-required $ref property must be wrapped in a { allOf: [ $ref ] }
+                            // A non-required $ref property must be wrapped in a { anyOf: [ $ref ] }
                             // object to allow the
                             // property to be marked as nullable
                             schema.setType(TYPE_OBJECT);
                             schema.set$ref(null);
-                            schema.setAllOf(List.of(new Schema().$ref($ref)));
+                            schema.setAnyOf(List.of(new Schema().$ref($ref)));
                           }
                           schema.setNullable(!isNameRequired);
                         });
@@ -578,7 +578,7 @@ private static Schema buildAspectRefResponseSchema(final String aspectName) {
         "systemMetadata",
         new Schema<>()
             .type(TYPE_OBJECT)
-            .allOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "SystemMetadata")))
+            .anyOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "SystemMetadata")))
             .description("System metadata for the aspect.")
             .nullable(true));
     return result;
@@ -595,7 +595,7 @@ private static Schema buildAspectRefRequestSchema(final String aspectName) {
         "systemMetadata",
         new Schema<>()
             .type(TYPE_OBJECT)
-            .allOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "SystemMetadata")))
+            .anyOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "SystemMetadata")))
             .description("System metadata for the aspect.")
             .nullable(true));
 
@@ -681,7 +681,7 @@ private static Schema buildEntityBatchGetRequestSchema(
   }
 
   private static Schema buildAspectRef(final String aspect, final boolean withSystemMetadata) {
-    // A non-required $ref property must be wrapped in a { allOf: [ $ref ] }
+    // A non-required $ref property must be wrapped in a { anyOf: [ $ref ] }
     // object to allow the
     // property to be marked as nullable
     final Schema result = new Schema<>();
@@ -697,7 +697,7 @@ private static Schema buildAspectRef(final String aspect, final boolean withSyst
       internalRef =
           String.format(FORMAT_PATH_DEFINITIONS, toUpperFirst(aspect), ASPECT_REQUEST_SUFFIX);
     }
-    result.setAllOf(List.of(new Schema().$ref(internalRef)));
+    result.setAnyOf(List.of(new Schema().$ref(internalRef)));
     return result;
   }
 

metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java

@@ -61,12 +61,12 @@ public void testOpenApiSpecBuilder() throws Exception {
     assertFalse(requiredNames.contains("name"));
     assertTrue(name.getNullable());
 
-    // Assert non-required $ref properties are replaced by nullable { allOf: [ $ref ] } objects
+    // Assert non-required $ref properties are replaced by nullable { anyOf: [ $ref ] } objects
     Schema created = properties.get("created");
     assertFalse(requiredNames.contains("created"));
     assertEquals("object", created.getType());
     assertNull(created.get$ref());
-    assertEquals(List.of(new Schema().$ref("#/components/schemas/TimeStamp")), created.getAllOf());
+    assertEquals(List.of(new Schema().$ref("#/components/schemas/TimeStamp")), created.getAnyOf());
     assertTrue(created.getNullable());
 
     // Assert systemMetadata property on response schema is optional.
@@ -81,7 +81,7 @@ public void testOpenApiSpecBuilder() throws Exception {
     assertNull(systemMetadata.get$ref());
     assertEquals(
         List.of(new Schema().$ref("#/components/schemas/SystemMetadata")),
-        systemMetadata.getAllOf());
+        systemMetadata.getAnyOf());
     assertTrue(systemMetadata.getNullable());
 
     // Assert enum property is string.