metadata-models 54.0.1 -> 58.0.1

gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json

@@ -1038,12 +1038,16 @@
     "name" : "Condition",
     "namespace" : "com.linkedin.metadata.query",
     "doc" : "The matching condition in a filter criterion",
-    "symbols" : [ "EQUAL", "START_WITH", "END_WITH", "CONTAIN" ],
+    "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ],
     "symbolDocs" : {
-      "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
-      "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView",
+      "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile",
       "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
-      "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile"
+      "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
+      "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
+      "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
+      "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3",
+      "LESS_THAN_OR_EQUAL_TO" : "Represent the relation less than or equal to, e.g. ownerCount <= 3",
+      "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView"
     }
   }, {
     "type" : "record",

gms/api/src/main/snapshot/com.linkedin.identity.corpGroups.snapshot.json

@@ -172,12 +172,16 @@
     "name" : "Condition",
     "namespace" : "com.linkedin.metadata.query",
     "doc" : "The matching condition in a filter criterion",
-    "symbols" : [ "EQUAL", "START_WITH", "END_WITH", "CONTAIN" ],
+    "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ],
     "symbolDocs" : {
-      "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
-      "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView",
+      "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile",
       "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
-      "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile"
+      "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
+      "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
+      "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
+      "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3",
+      "LESS_THAN_OR_EQUAL_TO" : "Represent the relation less than or equal to, e.g. ownerCount <= 3",
+      "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView"
     }
   }, {
     "type" : "record",

gms/api/src/main/snapshot/com.linkedin.identity.corpUsers.snapshot.json

@@ -220,12 +220,16 @@
     "name" : "Condition",
     "namespace" : "com.linkedin.metadata.query",
     "doc" : "The matching condition in a filter criterion",
-    "symbols" : [ "EQUAL", "START_WITH", "END_WITH", "CONTAIN" ],
+    "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ],
     "symbolDocs" : {
-      "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
-      "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView",
+      "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile",
       "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event",
-      "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile"
+      "EQUAL" : "Represent the relation: field = value, e.g. platform = hdfs",
+      "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5",
+      "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
+      "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3",
+      "LESS_THAN_OR_EQUAL_TO" : "Represent the relation less than or equal to, e.g. ownerCount <= 3",
+      "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView"
     }
   }, {
     "type" : "record",

gms/impl/build.gradle

@@ -19,6 +19,7 @@ dependencies {
 
   testCompile externalDependency.parseqTest
   testCompile externalDependency.mockito
+  testCompile externalDependency.testng
 }
 
 // Generate IDLs

metadata-dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/ImmutableLocalDAO.java

@@ -83,7 +83,7 @@ public static <URN extends Urn, ASPECT extends RecordTemplate> Map<URN, ASPECT>
 
   @Override
   @Nonnull
-  public <ASPECT extends RecordTemplate> Optional<RecordTemplate> add(@Nonnull URN urn, @Nonnull Class<ASPECT> aspectClass,
+  public <ASPECT extends RecordTemplate> RecordTemplate add(@Nonnull URN urn, @Nonnull Class<ASPECT> aspectClass,
       @Nonnull Function<Optional<RecordTemplate>, RecordTemplate> updateLambda, @Nonnull AuditStamp auditStamp,
       int maxTransactionRetry) {
     throw new UnsupportedOperationException("Not supported by immutable DAO");

metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/search/ESSearchDAO.java

@@ -45,6 +45,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
+import static com.linkedin.metadata.dao.utils.SearchUtils.*;
+
+
 /**
  * A search DAO for Elasticsearch backend.
  */
@@ -119,14 +122,44 @@ public SearchResult<DOCUMENT> search(@Nonnull String input, @Nullable Filter pos
 
   @Override
   @Nonnull
-  public SearchResult<DOCUMENT> filter(@Nullable Filter filters, @Nullable SortCriterion sortCriterion, int from, int size) {
-
-    final Map<String, String> requestMap = SearchUtils.getRequestMap(filters);
-    final SearchRequest searchRequest = ESUtils.getFilteredSearchQuery(requestMap, sortCriterion, from, size);
+  public SearchResult<DOCUMENT> filter(@Nullable Filter filters, @Nullable SortCriterion sortCriterion,
+      int from, int size) {
 
+    final SearchRequest searchRequest = getFilteredSearchQuery(filters, sortCriterion, from, size);
     return executeAndExtract(searchRequest, from, size);
   }
 
+  /**
+   * Returns a {@link SearchRequest} given filters to be applied to search query and sort criterion to be applied to search results
+   *
+   * @param filters {@link Filter} list of conditions with fields and values
+   * @param sortCriterion {@link SortCriterion} to be applied to the search results
+   * @param from index to start the search from
+   * @param size the number of search hits to return
+   * @return {@link SearchRequest} that contains the filtered query
+   */
+  @Nonnull
+  SearchRequest getFilteredSearchQuery(@Nullable Filter filters, @Nullable SortCriterion sortCriterion,
+      int from, int size) {
+
+    final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+    if (filters != null) {
+      filters.getCriteria().forEach(criterion -> {
+            if (!criterion.getValue().trim().isEmpty()) {
+              boolQueryBuilder.filter(getQueryBuilderFromCriterion(criterion));
+            }
+          });
+    }
+    final SearchRequest searchRequest = new SearchRequest(_config.getIndexName());
+    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+    searchSourceBuilder.query(boolQueryBuilder);
+    searchSourceBuilder.from(from).size(size);
+    ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
+    searchRequest.source(searchSourceBuilder);
+
+    return searchRequest;
+  }
+
   /**
    * Constructs the search query based on the query request
    *

metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/ESUtils.java

@@ -5,7 +5,6 @@
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -48,32 +47,6 @@ public static BoolQueryBuilder buildFilterQuery(@Nonnull Map<String, String> req
     return boolFilter;
   }
 
-  /**
-   * Returns a {@link SearchRequest} given filters to be applied to search query and sort criterion to be applied to search results
-   *
-   * @param requestMap search request map with fields and values
-   * @param sortCriterion {@link SortCriterion} to be applied to the search results
-   * @param from index to start the search from
-   * @param size the number of search hits to return
-   * @return {@link SearchRequest} that contains the filtered query
-   */
-  @Nonnull
-  public static SearchRequest getFilteredSearchQuery(@Nonnull Map<String, String> requestMap, @Nullable SortCriterion sortCriterion, int from, int size) {
-    final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
-    for (Map.Entry<String, String> entry : requestMap.entrySet()) {
-      if (!entry.getValue().trim().isEmpty()) {
-        boolQueryBuilder.filter(QueryBuilders.termsQuery(entry.getKey(), entry.getValue().trim().split("\\s*,\\s*")));
-      }
-    }
-    final SearchRequest searchRequest = new SearchRequest();
-    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-    searchSourceBuilder.query(boolQueryBuilder);
-    searchSourceBuilder.from(from).size(size);
-    buildSortOrder(searchSourceBuilder, sortCriterion);
-    searchRequest.source(searchSourceBuilder);
-    return searchRequest;
-  }
-
   /**
    * Populates source field of search query with the sort order as per the criterion provided
    *

metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java

@@ -1,5 +1,6 @@
 package com.linkedin.metadata.dao.utils;
 
+import com.linkedin.metadata.query.Condition;
 import com.linkedin.metadata.query.Criterion;
 import com.linkedin.metadata.query.CriterionArray;
 import com.linkedin.metadata.query.Filter;
@@ -13,6 +14,8 @@
 import javax.annotation.Nullable;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 
 
 @Slf4j
@@ -44,6 +47,30 @@ public static Map<String, String> getRequestMap(@Nullable Filter requestParams)
     return requestParams.getCriteria().stream().collect(Collectors.toMap(Criterion::getField, Criterion::getValue));
   }
 
+  /**
+   * Builds search query using criterion
+   *
+   * @param criterion {@link Criterion} single criterion which contains field, value and a comparison operator
+   * @return QueryBuilder
+   */
+  @Nonnull
+  public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull Criterion criterion) {
+    final Condition condition = criterion.getCondition();
+    if (condition == Condition.EQUAL) {
+      return QueryBuilders.termsQuery(criterion.getField(), criterion.getValue().trim().split("\\s*,\\s*"));
+    } else if (condition == Condition.GREATER_THAN) {
+      return QueryBuilders.rangeQuery(criterion.getField()).gt(criterion.getValue().trim());
+    } else if (condition == Condition.GREATER_THAN_OR_EQUAL_TO) {
+      return QueryBuilders.rangeQuery(criterion.getField()).gte(criterion.getValue().trim());
+    } else if (condition == Condition.LESS_THAN) {
+      return QueryBuilders.rangeQuery(criterion.getField()).lt(criterion.getValue().trim());
+    } else if (condition == Condition.LESS_THAN_OR_EQUAL_TO) {
+      return QueryBuilders.rangeQuery(criterion.getField()).lte(criterion.getValue().trim());
+    }
+
+    throw new IllegalArgumentException("Unsupported condition: " + condition);
+  }
+
   /**
    * Converts a requestMap to a filter
    *

metadata-dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/ESUtilsTest.java

@@ -2,12 +2,8 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.linkedin.metadata.dao.utils.ESUtils;
-import com.linkedin.metadata.query.SortCriterion;
-import com.linkedin.metadata.query.SortOrder;
-import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
-import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.testng.annotations.Test;
 
@@ -39,26 +35,6 @@ public void testBuildFilterQuery() throws Exception {
     assertEquals(queryBuilder.toString(), loadJsonFromResource("filterQuery/ComplexFilterQuery.json"));
   }
 
-  @Test
-  public void testGetFilteredSearchQuery() throws IOException {
-    int from = 0;
-    int size = 10;
-    Map<String, String> requestMap = ImmutableMap.of("key1", "value1, value2 ", "key2", "value3", "key3", " ");
-    SortCriterion sortCriterion = new SortCriterion().setOrder(SortOrder.ASCENDING).setField("urn");
-
-    // Test 1: sort order provided
-    SearchRequest searchRequest = ESUtils.getFilteredSearchQuery(requestMap, sortCriterion, from, size);
-    assertEquals(searchRequest.source().toString(), loadJsonFromResource("SortByUrnTermsFilterQuery.json"));
-
-    // Test 2: no sort order provided, default is used.
-    searchRequest = ESUtils.getFilteredSearchQuery(requestMap, null, from, size);
-    assertEquals(searchRequest.source().toString(), loadJsonFromResource("DefaultSortTermsFilterQuery.json"));
-
-    // Test 3: empty request map provided
-    searchRequest = ESUtils.getFilteredSearchQuery(Collections.emptyMap(), sortCriterion, from, size);
-    assertEquals(searchRequest.source().toString(), loadJsonFromResource("EmptyFilterQuery.json"));
-  }
-
   @Test
   public void testEscapeReservedCharacters() {
     assertEquals(escapeReservedCharacters("foobar"), "foobar");

metadata-dao-impl/elasticsearch-dao/src/test/java/com/linkedin/metadata/dao/browse/BrowseDAOTest.java

@@ -20,13 +20,13 @@
 public class BrowseDAOTest {
   private BaseBrowseConfig _browseConfig;
   private RestHighLevelClient _mockClient;
-  private ESBrowseDAO _mockBrowseDAO;
+  private ESBrowseDAO _browseDAO;
 
   @BeforeMethod
   public void setup() {
     _browseConfig = new TestBrowseConfig();
     _mockClient = mock(RestHighLevelClient.class);
-    _mockBrowseDAO = new ESBrowseDAO(_mockClient, _browseConfig);
+    _browseDAO = new ESBrowseDAO(_mockClient, _browseConfig);
   }
 
   @Test
@@ -79,15 +79,15 @@ public void testGetBrowsePath() throws Exception {
     when(mockSearchHits.getHits()).thenReturn(new SearchHit[0]);
     when(mockSearchResponse.getHits()).thenReturn(mockSearchHits);
     when(_mockClient.search(any())).thenReturn(mockSearchResponse);
-    assertEquals(_mockBrowseDAO.getBrowsePaths(dummyUrn).size(), 0);
+    assertEquals(_browseDAO.getBrowsePaths(dummyUrn).size(), 0);
 
     // Test the case of single search hit & browsePaths field doesn't exist
     when(mockSourceMap.containsKey(_browseConfig.getBrowsePathFieldName())).thenReturn(false);
     when(mockSearchHit.getSourceAsMap()).thenReturn(mockSourceMap);
     when(mockSearchHits.getHits()).thenReturn(new SearchHit[]{mockSearchHit});
     when(mockSearchResponse.getHits()).thenReturn(mockSearchHits);
     when(_mockClient.search(any())).thenReturn(mockSearchResponse);
-    assertEquals(_mockBrowseDAO.getBrowsePaths(dummyUrn).size(), 0);
+    assertEquals(_browseDAO.getBrowsePaths(dummyUrn).size(), 0);
 
     // Test the case of single search hit & browsePaths field exists
     when(mockSourceMap.containsKey(_browseConfig.getBrowsePathFieldName())).thenReturn(true);
@@ -96,7 +96,7 @@ public void testGetBrowsePath() throws Exception {
     when(mockSearchHits.getHits()).thenReturn(new SearchHit[]{mockSearchHit});
     when(mockSearchResponse.getHits()).thenReturn(mockSearchHits);
     when(_mockClient.search(any())).thenReturn(mockSearchResponse);
-    assertEquals(_mockBrowseDAO.getBrowsePaths(dummyUrn).size(), 1);
-    assertEquals(_mockBrowseDAO.getBrowsePaths(dummyUrn).get(0), "foo");
+    assertEquals(_browseDAO.getBrowsePaths(dummyUrn).size(), 1);
+    assertEquals(_browseDAO.getBrowsePaths(dummyUrn).get(0), "foo");
   }
 }