Create index field mapping aliases (#102)

* update proto for field aliases * add field mapping aliases
zuliaio · May 8, 2023 · 96e26d9 · 96e26d9
1 parent 7b08b1b
commit 96e26d9
Show file tree

Hide file tree

Showing 10 changed files with 332 additions and 31 deletions.
diff --git a/zulia-analyzer/src/main/java/io/zulia/server/config/ServerIndexConfig.java b/zulia-analyzer/src/main/java/io/zulia/server/config/ServerIndexConfig.java
@@ -3,6 +3,7 @@
 import com.google.protobuf.ByteString;
 import io.zulia.DefaultAnalyzers;
 import io.zulia.ZuliaFieldConstants;
+import io.zulia.message.ZuliaIndex;
 import io.zulia.message.ZuliaIndex.AnalyzerSettings;
 import io.zulia.message.ZuliaIndex.AnalyzerSettings.Filter;
 import io.zulia.message.ZuliaIndex.AnalyzerSettings.Tokenizer;
@@ -17,6 +18,7 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
@@ -36,6 +38,8 @@ public class ServerIndexConfig {
 
 	private List<ZuliaServiceOuterClass.QueryRequest> warmingSearches;
 
+	private ConcurrentHashMap<String, Set<String>> fieldMappingToFields;
+
 	public ServerIndexConfig(IndexSettings indexSettings) {
 		configure(indexSettings);
 	}
@@ -113,6 +117,22 @@ public void configure(IndexSettings indexSettings) {
 			}
 		}
 
+		this.fieldMappingToFields = new ConcurrentHashMap<>();
+
+		for (ZuliaIndex.FieldMapping fieldMapping : indexSettings.getFieldMappingList()) {
+
+			Set<String> matchingFields = new HashSet<>();
+			for (String fieldOrFieldPattern : fieldMapping.getFieldOrFieldPatternList()) {
+				matchingFields.addAll(getMatchingIndexFields(fieldOrFieldPattern, false));
+			}
+
+			if (fieldMapping.getIncludeSelf()) {
+				matchingFields.add(fieldMapping.getAlias());
+			}
+
+			fieldMappingToFields.put(fieldMapping.getAlias(), matchingFields);
+		}
+
 	}
 
 	private void populateAnalyzers() {
@@ -128,17 +148,22 @@ private void populateAnalyzers() {
 						.addFilter(Filter.STOPWORDS).addFilter(Filter.ENGLISH_MIN_STEM).build());
 
 		analyzerMap.put(DefaultAnalyzers.TWO_TWO_SHINGLE,
-				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.TWO_TWO_SHINGLE).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.TWO_TWO_SHINGLE).build());
+				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.TWO_TWO_SHINGLE).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE)
+						.addFilter(Filter.TWO_TWO_SHINGLE).build());
 		analyzerMap.put(DefaultAnalyzers.THREE_THREE_SHINGLE,
-				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.THREE_THREE_SHINGLE).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.THREE_THREE_SHINGLE).build());
+				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.THREE_THREE_SHINGLE).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE)
+						.addFilter(Filter.THREE_THREE_SHINGLE).build());
 
 		analyzerMap.put(DefaultAnalyzers.LC_CONCAT_ALL,
-				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.LC_CONCAT_ALL).setTokenizer(Tokenizer.KEYWORD).addFilter(Filter.LOWERCASE).addFilter(Filter.CONCAT_ALL).build());
+				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.LC_CONCAT_ALL).setTokenizer(Tokenizer.KEYWORD).addFilter(Filter.LOWERCASE)
+						.addFilter(Filter.CONCAT_ALL).build());
 
 		analyzerMap.put(DefaultAnalyzers.KSTEMMED,
-				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.KSTEMMED).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.STOPWORDS).addFilter(Filter.KSTEM).build());
+				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.KSTEMMED).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE)
+						.addFilter(Filter.STOPWORDS).addFilter(Filter.KSTEM).build());
 		analyzerMap.put(DefaultAnalyzers.LSH,
-				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.LSH).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.ASCII_FOLDING).addFilter(Filter.KSTEM).addFilter(Filter.STOPWORDS).addFilter(Filter.FIVE_FIVE_SHINGLE)
+				AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.LSH).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE)
+						.addFilter(Filter.ASCII_FOLDING).addFilter(Filter.KSTEM).addFilter(Filter.STOPWORDS).addFilter(Filter.FIVE_FIVE_SHINGLE)
 						.addFilter(Filter.MINHASH).build());
 
 		for (AnalyzerSettings analyzerSettings : indexSettings.getAnalyzerSettingsList()) {
@@ -187,7 +212,10 @@ public int getRAMBufferMB() {
 	}
 
 	public Set<String> getMatchingFields(String field) {
+		return getMatchingIndexFields(field, true);
+	}
 
+	private Set<String> getMatchingIndexFields(String field, boolean includeAliases) {
 		if (field.contains("*")) {
 
 			field = ("\\Q" + field + "\\E").replace("*", "\\E.*\\Q");
@@ -200,10 +228,23 @@ public Set<String> getMatchingFields(String field) {
 					matchingFieldNames.add(indexFieldName);
 				}
 			}
+
+			if (includeAliases) {
+				for (String alias : fieldMappingToFields.keySet()) {
+					if (pattern.matcher(alias).matches()) {
+						matchingFieldNames.addAll(fieldMappingToFields.get(alias));
+					}
+				}
+			}
+
 			return matchingFieldNames;
 		}
-		return Set.of(field);
 
+		if (includeAliases && fieldMappingToFields.containsKey(field)) {
+			return fieldMappingToFields.get(field);
+		}
+
+		return Set.of(field);
 	}
 
 	public List<ZuliaServiceOuterClass.QueryRequest> getWarmingSearches() {

diff --git a/zulia-client/src/main/java/io/zulia/client/command/UpdateIndex.java b/zulia-client/src/main/java/io/zulia/client/command/UpdateIndex.java
@@ -2,11 +2,13 @@
 
 import io.zulia.client.command.base.SimpleCommand;
 import io.zulia.client.command.base.SingleIndexRoutableCommand;
+import io.zulia.client.command.builder.FieldMappingBuilder;
 import io.zulia.client.command.builder.Search;
 import io.zulia.client.pool.ZuliaConnection;
 import io.zulia.client.result.UpdateIndexResult;
 import io.zulia.fields.FieldConfigBuilder;
 import io.zulia.message.ZuliaIndex;
+import io.zulia.message.ZuliaIndex.FieldMapping;
 import io.zulia.message.ZuliaIndex.UpdateIndexSettings;
 import io.zulia.message.ZuliaIndex.UpdateIndexSettings.Operation.OperationType;
 import io.zulia.message.ZuliaServiceOuterClass.QueryRequest;
@@ -56,6 +58,9 @@ public class UpdateIndex extends SimpleCommand<UpdateIndexRequest, UpdateIndexRe
 
 	private List<QueryRequest> warmingSearches = Collections.emptyList();
 
+	private final UpdateIndexSettings.Operation.Builder fieldMappingOperation = UpdateIndexSettings.Operation.newBuilder();
+	private List<ZuliaIndex.FieldMapping> fieldMappingList = Collections.emptyList();
+
 	public UpdateIndex(String indexName) {
 		this.indexName = indexName;
 	}
@@ -168,6 +173,54 @@ public UpdateIndex mergeFieldConfig(List<ZuliaIndex.FieldConfig> fieldConfigs) {
 		return this;
 	}
 
+	public UpdateIndex removeFieldMappingByAlias(String... aliasesToRemove) {
+		return removeFieldMappingByAlias(Arrays.asList(aliasesToRemove));
+	}
+
+	public UpdateIndex removeFieldMappingByAlias(Collection<String> aliasesToRemove) {
+		this.fieldMappingOperation.setEnable(true);
+		this.fieldMappingOperation.addAllRemovedKeys(aliasesToRemove);
+		return this;
+	}
+
+	public UpdateIndex replaceFieldMapping(FieldMapping... fieldMapping) {
+		return replaceFieldMapping(Arrays.asList(fieldMapping));
+	}
+
+	public UpdateIndex replaceFieldMapping(FieldMappingBuilder... fieldMapping) {
+		return replaceFieldMapping(Arrays.stream(fieldMapping).map(FieldMappingBuilder::getFieldMapping).collect(Collectors.toList()));
+	}
+
+	public UpdateIndex replaceFieldMapping(List<FieldMapping> fieldMappings) {
+		if (fieldMappings == null) {
+			fieldMappings = Collections.emptyList();
+		}
+
+		this.fieldMappingOperation.setEnable(true);
+		this.fieldMappingOperation.setOperationType(OperationType.REPLACE);
+		this.fieldMappingList = fieldMappings;
+		return this;
+	}
+
+	public UpdateIndex mergeFieldMapping(FieldMapping... fieldMappings) {
+		return mergeFieldMapping(Arrays.asList(fieldMappings));
+	}
+
+	public UpdateIndex mergeFieldMapping(FieldMappingBuilder... mergeFieldMappings) {
+		return mergeFieldMapping(Arrays.stream(mergeFieldMappings).map(FieldMappingBuilder::getFieldMapping).toList());
+	}
+
+	public UpdateIndex mergeFieldMapping(List<FieldMapping> fieldMappings) {
+		if (fieldMappings == null || fieldMappings.isEmpty()) {
+			throw new IllegalArgumentException("Cannot merge null or empty field mappings");
+		}
+
+		this.fieldMappingOperation.setEnable(true);
+		this.fieldMappingOperation.setOperationType(OperationType.MERGE);
+		this.fieldMappingList = fieldMappings;
+		return this;
+	}
+
 	public Double getRequestFactor() {
 		return requestFactor;
 	}
@@ -378,6 +431,9 @@ public UpdateIndexRequest getRequest() {
 		updateIndexSettings.addAllFieldConfig(fieldConfigList);
 		updateIndexSettings.setFieldConfigOperation(fieldConfigOperation);
 
+		updateIndexSettings.addAllFieldMapping(fieldMappingList);
+		updateIndexSettings.setFieldMappingOperation(fieldMappingOperation);
+
 		updateIndexSettings.addAllWarmingSearches(warmingSearches.stream().map(QueryRequest::toByteString).toList());
 		updateIndexSettings.setWarmingSearchesOperation(warmingSearchOperation);
 

diff --git a/zulia-client/src/main/java/io/zulia/client/command/builder/FieldMapping.java b/zulia-client/src/main/java/io/zulia/client/command/builder/FieldMapping.java
@@ -0,0 +1,40 @@
+package io.zulia.client.command.builder;
+
+import io.zulia.message.ZuliaIndex;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public class FieldMapping implements FieldMappingBuilder {
+
+	private final String alias;
+	private final Set<String> fields;
+	private boolean includeSelf;
+
+	public FieldMapping(String alias) {
+		this.alias = alias;
+		this.fields = new LinkedHashSet<>();
+	}
+
+	public FieldMapping addMappedFields(String... fieldOrFieldPatterns) {
+		for (String fieldOrFieldPattern : fieldOrFieldPatterns) {
+			fields.add(fieldOrFieldPattern);
+		}
+		return this;
+	}
+
+	public FieldMapping includeSelf() {
+		this.includeSelf = true;
+		return this;
+	}
+
+	public FieldMapping excludeSelf() {
+		this.includeSelf = false;
+		return this;
+	}
+
+	@Override
+	public ZuliaIndex.FieldMapping getFieldMapping() {
+		return ZuliaIndex.FieldMapping.newBuilder().setAlias(alias).addAllFieldOrFieldPattern(fields).setIncludeSelf(includeSelf).build();
+	}
+}
diff --git a/zulia-client/src/main/java/io/zulia/client/command/builder/FieldMappingBuilder.java b/zulia-client/src/main/java/io/zulia/client/command/builder/FieldMappingBuilder.java
@@ -0,0 +1,8 @@
+package io.zulia.client.command.builder;
+
+import io.zulia.message.ZuliaIndex.FieldMapping;
+
+public interface FieldMappingBuilder {
+
+	FieldMapping getFieldMapping();
+}