-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(structuredProperties): add hide property and show as badge valid…
…ators (#4484)
- Loading branch information
1 parent
f1ef4f8
commit 3b0e89a
Showing
8 changed files
with
576 additions
and
70 deletions.
There are no files selected for viewing
35 changes: 35 additions & 0 deletions
35
entity-registry/src/main/java/com/linkedin/metadata/entity/GenericScrollIterator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package com.linkedin.metadata.entity; | ||
|
|
||
| import com.linkedin.metadata.query.filter.Filter; | ||
| import com.linkedin.metadata.search.ScrollResult; | ||
| import java.util.Iterator; | ||
| import java.util.List; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.Builder; | ||
|
|
||
| /** | ||
| * Fetches pages of structured properties which have been applied to an entity urn with a specified | ||
| * filter | ||
| */ | ||
| @Builder | ||
| public class GenericScrollIterator implements Iterator<ScrollResult> { | ||
| @Nonnull private final Filter filter; | ||
| @Nonnull private final List<String> entities; | ||
| @Nonnull private final SearchRetriever searchRetriever; | ||
| private int count; | ||
| @Builder.Default private String scrollId = null; | ||
| @Builder.Default private boolean started = false; | ||
|
|
||
| @Override | ||
| public boolean hasNext() { | ||
| return !started || scrollId != null; | ||
| } | ||
|
|
||
| @Override | ||
| public ScrollResult next() { | ||
| started = true; | ||
| ScrollResult result = searchRetriever.scroll(entities, filter, scrollId, count); | ||
| scrollId = result.getScrollId(); | ||
| return result; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
76 changes: 76 additions & 0 deletions
76
...main/java/com/linkedin/metadata/structuredproperties/util/EntityWithPropertyIterator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package com.linkedin.metadata.structuredproperties.util; | ||
|
|
||
| import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX; | ||
| import static com.linkedin.metadata.utils.CriterionUtils.buildExistsCriterion; | ||
|
|
||
| import com.linkedin.common.urn.Urn; | ||
| import com.linkedin.metadata.entity.SearchRetriever; | ||
| import com.linkedin.metadata.models.StructuredPropertyUtils; | ||
| import com.linkedin.metadata.query.filter.ConjunctiveCriterion; | ||
| import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; | ||
| import com.linkedin.metadata.query.filter.Criterion; | ||
| import com.linkedin.metadata.query.filter.CriterionArray; | ||
| import com.linkedin.metadata.query.filter.Filter; | ||
| import com.linkedin.metadata.search.ScrollResult; | ||
| import com.linkedin.structured.StructuredPropertyDefinition; | ||
| import java.util.Collections; | ||
| import java.util.Iterator; | ||
| import java.util.List; | ||
| import java.util.stream.Collectors; | ||
| import javax.annotation.Nonnull; | ||
| import javax.annotation.Nullable; | ||
| import lombok.Builder; | ||
|
|
||
| /** Fetches pages of entity urns which have a value for the given structured property definition */ | ||
| @Builder | ||
| public class EntityWithPropertyIterator implements Iterator<ScrollResult> { | ||
| @Nonnull private final Urn propertyUrn; | ||
| @Nullable private final StructuredPropertyDefinition definition; | ||
| @Nonnull private final SearchRetriever searchRetriever; | ||
| private int count; | ||
| @Builder.Default private String scrollId = null; | ||
| @Builder.Default private boolean started = false; | ||
|
|
||
| private List<String> getEntities() { | ||
| if (definition != null && definition.getEntityTypes() != null) { | ||
| return definition.getEntityTypes().stream() | ||
| .map(StructuredPropertyUtils::getValueTypeId) | ||
| .collect(Collectors.toList()); | ||
| } else { | ||
| return Collections.emptyList(); | ||
| } | ||
| } | ||
|
|
||
| private Filter getFilter() { | ||
| Filter propertyFilter = new Filter(); | ||
| final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray(); | ||
| final ConjunctiveCriterion conjunction = new ConjunctiveCriterion(); | ||
| final CriterionArray andCriterion = new CriterionArray(); | ||
|
|
||
| // Cannot rely on automatic field name since the definition is deleted | ||
| final Criterion propertyExistsCriterion = | ||
| buildExistsCriterion( | ||
| STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX | ||
| + StructuredPropertyUtils.toElasticsearchFieldName(propertyUrn, definition)); | ||
|
|
||
| andCriterion.add(propertyExistsCriterion); | ||
| conjunction.setAnd(andCriterion); | ||
| disjunction.add(conjunction); | ||
| propertyFilter.setOr(disjunction); | ||
|
|
||
| return propertyFilter; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean hasNext() { | ||
| return !started || scrollId != null; | ||
| } | ||
|
|
||
| @Override | ||
| public ScrollResult next() { | ||
| started = true; | ||
| ScrollResult result = searchRetriever.scroll(getEntities(), getFilter(), scrollId, count); | ||
| scrollId = result.getScrollId(); | ||
| return result; | ||
| } | ||
| } |
64 changes: 64 additions & 0 deletions
64
...ain/java/com/linkedin/metadata/structuredproperties/validation/HidePropertyValidator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| package com.linkedin.metadata.structuredproperties.validation; | ||
|
|
||
| import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME; | ||
|
|
||
| import com.google.common.annotations.VisibleForTesting; | ||
| import com.linkedin.metadata.aspect.RetrieverContext; | ||
| import com.linkedin.metadata.aspect.batch.BatchItem; | ||
| import com.linkedin.metadata.aspect.batch.ChangeMCP; | ||
| import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; | ||
| import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; | ||
| import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; | ||
| import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection; | ||
| import com.linkedin.metadata.models.StructuredPropertyUtils; | ||
| import com.linkedin.structured.StructuredPropertySettings; | ||
| import java.util.Collection; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.Getter; | ||
| import lombok.Setter; | ||
| import lombok.experimental.Accessors; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.jetbrains.annotations.NotNull; | ||
|
|
||
| @Setter | ||
| @Getter | ||
| @Slf4j | ||
| @Accessors(chain = true) | ||
| public class HidePropertyValidator extends AspectPayloadValidator { | ||
|
|
||
| @Nonnull private AspectPluginConfig config; | ||
|
|
||
| @Override | ||
| protected Stream<AspectValidationException> validateProposedAspects( | ||
| @NotNull Collection<? extends BatchItem> mcpItems, | ||
| @NotNull RetrieverContext retrieverContext) { | ||
| return validateSettingsUpserts( | ||
| mcpItems.stream() | ||
| .filter(i -> STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME.equals(i.getAspectName())) | ||
| .collect(Collectors.toList())); | ||
| } | ||
|
|
||
| @Override | ||
| protected Stream<AspectValidationException> validatePreCommitAspects( | ||
| @NotNull Collection<ChangeMCP> changeMCPs, @NotNull RetrieverContext retrieverContext) { | ||
| return Stream.empty(); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| public static Stream<AspectValidationException> validateSettingsUpserts( | ||
| @NotNull Collection<? extends BatchItem> mcpItems) { | ||
| ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection(); | ||
| for (BatchItem mcpItem : mcpItems) { | ||
| StructuredPropertySettings structuredPropertySettings = | ||
| mcpItem.getAspect(StructuredPropertySettings.class); | ||
| boolean isValid = | ||
| StructuredPropertyUtils.validatePropertySettings(structuredPropertySettings, false); | ||
| if (!isValid) { | ||
| exceptions.addException(mcpItem, StructuredPropertyUtils.INVALID_SETTINGS_MESSAGE); | ||
| } | ||
| } | ||
| return exceptions.streamAllExceptions(); | ||
| } | ||
| } |
Oops, something went wrong.