feat(structuredProps) Add frontend for managing structured props and …

…filtering by them (#12097)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -67,6 +67,7 @@
 import com.linkedin.datahub.graphql.generated.EntityPath;
 import com.linkedin.datahub.graphql.generated.EntityRelationship;
 import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
+import com.linkedin.datahub.graphql.generated.FacetMetadata;
 import com.linkedin.datahub.graphql.generated.ForeignKeyConstraint;
 import com.linkedin.datahub.graphql.generated.FormActorAssignment;
 import com.linkedin.datahub.graphql.generated.FreshnessContract;
@@ -1474,6 +1475,19 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder
                     "entity",
                     new EntityTypeResolver(
                         entityTypes, (env) -> ((BrowsePathEntry) env.getSource()).getEntity())))
+        .type(
+            "FacetMetadata",
+            typeWiring ->
+                typeWiring.dataFetcher(
+                    "entity",
+                    new EntityTypeResolver(
+                        entityTypes,
+                        (env) -> {
+                          FacetMetadata facetMetadata = env.getSource();
+                          return facetMetadata.getEntity() != null
+                              ? facetMetadata.getEntity()
+                              : null;
+                        })))
         .type(
             "LineageRelationship",
             typeWiring ->

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java

@@ -339,6 +339,11 @@ public static boolean canManageStructuredProperties(@Nonnull QueryContext contex
         context.getOperationContext(), PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE);
   }
 
+  public static boolean canViewStructuredPropertiesPage(@Nonnull QueryContext context) {
+    return AuthUtil.isAuthorized(
+        context.getOperationContext(), PoliciesConfig.VIEW_STRUCTURED_PROPERTIES_PAGE_PRIVILEGE);
+  }
+
   public static boolean canManageForms(@Nonnull QueryContext context) {
     return AuthUtil.isAuthorized(
         context.getOperationContext(), PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java

@@ -93,6 +93,10 @@ public CompletableFuture<AuthenticatedUser> get(DataFetchingEnvironment environm
                 BusinessAttributeAuthorizationUtils.canCreateBusinessAttribute(context));
             platformPrivileges.setManageBusinessAttributes(
                 BusinessAttributeAuthorizationUtils.canManageBusinessAttribute(context));
+            platformPrivileges.setManageStructuredProperties(
+                AuthorizationUtils.canManageStructuredProperties(context));
+            platformPrivileges.setViewStructuredPropertiesPage(
+                AuthorizationUtils.canViewStructuredPropertiesPage(context));
             // Construct and return authenticated user object.
             final AuthenticatedUser authUser = new AuthenticatedUser();
             authUser.setCorpUser(corpUser);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java

@@ -188,6 +188,7 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
             .setDataContractsEnabled(_featureFlags.isDataContractsEnabled())
             .setEditableDatasetNameEnabled(_featureFlags.isEditableDatasetNameEnabled())
             .setShowSeparateSiblings(_featureFlags.isShowSeparateSiblings())
+            .setShowManageStructuredProperties(_featureFlags.isShowManageStructuredProperties())
             .build();
 
     appConfig.setFeatureFlags(featureFlagsConfig);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java

@@ -66,11 +66,17 @@ public CompletableFuture<AggregateResults> get(DataFetchingEnvironment environme
 
           final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters());
 
-          final SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags());
+          final SearchFlags searchFlags =
+              input.getSearchFlags() != null
+                  ? mapInputFlags(context, input.getSearchFlags())
+                  : new SearchFlags();
 
           final List<String> facets =
               input.getFacets() != null && input.getFacets().size() > 0 ? input.getFacets() : null;
 
+          // do not include default facets if we're requesting any facets specifically
+          searchFlags.setIncludeDefaultFacets(facets == null || facets.size() <= 0);
+
           List<String> finalEntities =
               maybeResolvedView != null
                   ? SearchUtils.intersectEntityTypes(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java

@@ -2,19 +2,28 @@
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
 import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.*;
+import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.getEntityNames;
 
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
+import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput;
 import com.linkedin.datahub.graphql.generated.SearchResults;
 import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.query.SearchFlags;
+import com.linkedin.metadata.query.filter.Condition;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
+import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
+import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.search.SearchResult;
 import com.linkedin.metadata.service.ViewService;
+import com.linkedin.metadata.utils.CriterionUtils;
 import com.linkedin.view.DataHubViewInfo;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
@@ -64,24 +73,7 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
               ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters());
 
           SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags());
-          List<SortCriterion> sortCriteria;
-          if (input.getSortInput() != null) {
-            if (input.getSortInput().getSortCriteria() != null) {
-              sortCriteria =
-                  input.getSortInput().getSortCriteria().stream()
-                      .map(SearchUtils::mapSortCriterion)
-                      .collect(Collectors.toList());
-            } else {
-              sortCriteria =
-                  input.getSortInput().getSortCriterion() != null
-                      ? Collections.singletonList(
-                          mapSortCriterion(input.getSortInput().getSortCriterion()))
-                      : Collections.emptyList();
-            }
-
-          } else {
-            sortCriteria = Collections.emptyList();
-          }
+          List<SortCriterion> sortCriteria = SearchUtils.getSortCriteria(input.getSortInput());
 
           try {
             log.debug(
@@ -101,6 +93,14 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
               return SearchUtils.createEmptySearchResults(start, count);
             }
 
+            boolean shouldIncludeStructuredPropertyFacets =
+                input.getSearchFlags() != null
+                        && input.getSearchFlags().getIncludeStructuredPropertyFacets() != null
+                    ? input.getSearchFlags().getIncludeStructuredPropertyFacets()
+                    : false;
+            List<String> structuredPropertyFacets =
+                shouldIncludeStructuredPropertyFacets ? getStructuredPropertyFacets(context) : null;
+
             return UrnSearchResultsMapper.map(
                 context,
                 _entityClient.searchAcrossEntities(
@@ -113,7 +113,8 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
                         : baseFilter,
                     start,
                     count,
-                    sortCriteria));
+                    sortCriteria,
+                    structuredPropertyFacets));
           } catch (Exception e) {
             log.error(
                 "Failed to execute search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}",
@@ -133,4 +134,45 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
         this.getClass().getSimpleName(),
         "get");
   }
+
+  private List<String> getStructuredPropertyFacets(final QueryContext context) {
+    try {
+      SearchFlags searchFlags = new SearchFlags().setSkipCache(true);
+      SearchResult result =
+          _entityClient.searchAcrossEntities(
+              context.getOperationContext().withSearchFlags(flags -> searchFlags),
+              getEntityNames(ImmutableList.of(EntityType.STRUCTURED_PROPERTY)),
+              "*",
+              createStructuredPropertyFilter(),
+              0,
+              100,
+              Collections.emptyList(),
+              null);
+      return result.getEntities().stream()
+          .map(entity -> String.format("structuredProperties.%s", entity.getEntity().getId()))
+          .collect(Collectors.toList());
+    } catch (Exception e) {
+      log.error("Failed to get structured property facets to filter on", e);
+      return Collections.emptyList();
+    }
+  }
+
+  private Filter createStructuredPropertyFilter() {
+    return new Filter()
+        .setOr(
+            new ConjunctiveCriterionArray(
+                ImmutableList.of(
+                    new ConjunctiveCriterion()
+                        .setAnd(
+                            new CriterionArray(
+                                ImmutableList.of(
+                                    CriterionUtils.buildCriterion(
+                                        "filterStatus", Condition.EQUAL, "ENABLED")))),
+                    new ConjunctiveCriterion()
+                        .setAnd(
+                            new CriterionArray(
+                                ImmutableList.of(
+                                    CriterionUtils.buildCriterion(
+                                        "showInSearchFilters", Condition.EQUAL, "true")))))));
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java

@@ -22,6 +22,7 @@
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
 import com.linkedin.datahub.graphql.generated.SearchResults;
+import com.linkedin.datahub.graphql.generated.SearchSortInput;
 import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper;
 import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
 import com.linkedin.metadata.query.SearchFlags;
@@ -326,4 +327,25 @@ public static SearchResults createEmptySearchResults(final int start, final int
     result.setFacets(new ArrayList<>());
     return result;
   }
+
+  public static List<SortCriterion> getSortCriteria(@Nullable final SearchSortInput sortInput) {
+    List<SortCriterion> sortCriteria;
+    if (sortInput != null) {
+      if (sortInput.getSortCriteria() != null) {
+        sortCriteria =
+            sortInput.getSortCriteria().stream()
+                .map(SearchUtils::mapSortCriterion)
+                .collect(Collectors.toList());
+      } else {
+        sortCriteria =
+            sortInput.getSortCriterion() != null
+                ? Collections.singletonList(mapSortCriterion(sortInput.getSortCriterion()))
+                : new ArrayList<>();
+      }
+    } else {
+      sortCriteria = new ArrayList<>();
+    }
+
+    return sortCriteria;
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java

@@ -70,6 +70,9 @@ public static FacetMetadata mapFacet(
         aggregationFacets.stream()
             .map(facet -> facet.equals("entity") || facet.contains("_entityType"))
             .collect(Collectors.toList());
+    if (aggregationMetadata.getEntity() != null) {
+      facetMetadata.setEntity(UrnToEntityMapper.map(context, aggregationMetadata.getEntity()));
+    }
     facetMetadata.setField(aggregationMetadata.getName());
     facetMetadata.setDisplayName(
         Optional.ofNullable(aggregationMetadata.getDisplayName())

datahub-graphql-core/src/main/resources/app.graphql

@@ -156,6 +156,15 @@ type PlatformPrivileges {
   """
   manageBusinessAttributes: Boolean!
 
+  """
+  Whether the user can create, edit, and delete structured properties.
+  """
+  manageStructuredProperties: Boolean!
+
+  """
+  Whether the user can view the manage structured properties page.
+  """
+  viewStructuredPropertiesPage: Boolean!
 }
 
 """
@@ -517,6 +526,11 @@ type FeatureFlagsConfig {
   If turned on, all siblings will be separated with no way to get to a "combined" sibling view
   """
   showSeparateSiblings: Boolean!
+
+  """
+  If turned on, show the manage structured properties tab in the govern dropdown
+  """
+  showManageStructuredProperties: Boolean!
 }
 
 """

datahub-graphql-core/src/main/resources/search.graphql

@@ -167,6 +167,11 @@ input  SearchFlags {
   fields to include for custom Highlighting
   """
   customHighlightingFields: [String!]
+
+  """
+  Whether or not to fetch and request for structured property facets when doing a search
+  """
+  includeStructuredPropertyFacets: Boolean
 }
 
 """
@@ -872,6 +877,11 @@ type FacetMetadata {
   """
   displayName: String
 
+  """
+  Entity corresponding to the facet
+  """
+  entity: Entity
+
   """
   Aggregated search result counts by value of the field
   """

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java

@@ -471,7 +471,8 @@ private static EntityClient initMockEntityClient(
                 Mockito.eq(filter),
                 Mockito.eq(start),
                 Mockito.eq(limit),
-                Mockito.eq(Collections.emptyList())))
+                Mockito.eq(Collections.emptyList()),
+                Mockito.eq(null)))
         .thenReturn(result);
     return client;
   }
@@ -496,7 +497,8 @@ private static void verifyMockEntityClient(
             Mockito.eq(filter),
             Mockito.eq(start),
             Mockito.eq(limit),
-            Mockito.eq(Collections.emptyList()));
+            Mockito.eq(Collections.emptyList()),
+            Mockito.eq(null));
   }
 
   private static void verifyMockViewService(ViewService mockService, Urn viewUrn) {