From b4c12bac66c55453490714c459255d610ae4d285 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Mon, 16 Dec 2024 18:23:47 -0500 Subject: [PATCH 1/3] Attribute ColumnVisibility optimizations --- pom.xml | 2 +- .../datawave/query/attributes/Attribute.java | 31 +++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index a47f3d0d44a..65dce136d15 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ 11 UTF-8 1C - 2.1.2 + 2.1.4-SNAPSHOT 1.4.1.Final 1.0.0.Final 3.20.2 diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java index c27ff50c289..b4a90f89179 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java @@ -4,12 +4,15 @@ import java.io.DataOutput; import java.io.IOException; import java.util.Collection; +import java.util.Collections; +import java.util.Map; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.collections4.map.LRUMap; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; @@ -36,6 +39,7 @@ public abstract class Attribute> implements WritableComp protected Key metadata = null; protected boolean toKeep = true; // a flag denoting whether this attribute is to be kept in the returned results (transient or not) protected boolean fromIndex = true; // Assume attributes are from the index unless specified otherwise. + private static final Map visibilityCache = Collections.synchronizedMap(new LRUMap<>(5000)); public Attribute() {} @@ -50,12 +54,22 @@ public boolean isMetadataSet() { public ColumnVisibility getColumnVisibility() { if (isMetadataSet()) { - return metadata.getColumnVisibilityParsed(); + Text colVisTxt = metadata.getColumnVisibility(); + if (visibilityCache.containsKey(colVisTxt)) { + return visibilityCache.get(colVisTxt).deepCopy(); + } + ColumnVisibility newColvis = new ColumnVisibility(colVisTxt); + visibilityCache.put(colVisTxt, newColvis); + return newColvis.deepCopy(); } return Constants.EMPTY_VISIBILITY; } - public void setColumnVisibility(ColumnVisibility columnVisibility) { + public void setColumnVisibility(ColumnVisibility visibility) { + setColumnVisibility(new Text(visibility.getExpression())); + } + + private void setColumnVisibility(Text columnVisibility) { if (isMetadataSet()) { metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), metadata.getColumnQualifier(), columnVisibility, metadata.getTimestamp()); } else { @@ -82,7 +96,12 @@ public void setTimestamp(long ts) { * * Set the metadata. This should only be set here or from extended classes. */ - protected void setMetadata(ColumnVisibility vis, long ts) { + protected void setMetadata(ColumnVisibility visibility, long ts) { + Text vis = new Text(visibility.getExpression()); + setMetadata(vis, ts); + } + + private void setMetadata(Text vis, long ts) { if (isMetadataSet()) { metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), metadata.getColumnQualifier(), vis, ts); } else { @@ -191,7 +210,7 @@ protected void readMetadata(DataInput in) throws IOException { in.readFully(cvBytes); - this.setMetadata(new ColumnVisibility(cvBytes), in.readLong()); + this.setMetadata(new Text(cvBytes), in.readLong()); } else { this.clearMetadata(); } @@ -201,7 +220,7 @@ protected void readMetadata(Kryo kryo, Input input) { if (input.readBoolean()) { int size = input.readInt(true); - this.setMetadata(new ColumnVisibility(input.readBytes(size)), input.readLong()); + this.setMetadata(new Text(input.readBytes(size)), input.readLong()); } else { this.clearMetadata(); } @@ -301,7 +320,7 @@ protected static long sizeInBytes(String value) { if (value == null) { return 0; } else { - return 16 + roundUp(12 + (value.length() * 2)); + return 16 + roundUp(12 + (value.length() * 2L)); // 16 for int, array ref, and object overhead // 12 for array overhead } From bbc2390e9dfe9898fe8efd6ca309fda3a0d27dce Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 27 Dec 2024 14:10:52 -0500 Subject: [PATCH 2/3] swap LRUMap for Caffeine cache --- .../java/datawave/query/attributes/Attribute.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java index b4a90f89179..32102d6d6e7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java @@ -4,15 +4,12 @@ import java.io.DataOutput; import java.io.IOException; import java.util.Collection; -import java.util.Collections; -import java.util.Map; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.collections4.map.LRUMap; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; @@ -23,6 +20,8 @@ import com.esotericsoftware.kryo.KryoSerializable; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import datawave.query.Constants; import datawave.query.jexl.DatawaveJexlContext; @@ -39,7 +38,7 @@ public abstract class Attribute> implements WritableComp protected Key metadata = null; protected boolean toKeep = true; // a flag denoting whether this attribute is to be kept in the returned results (transient or not) protected boolean fromIndex = true; // Assume attributes are from the index unless specified otherwise. - private static final Map visibilityCache = Collections.synchronizedMap(new LRUMap<>(5000)); + private static final LoadingCache visibilityCache = Caffeine.newBuilder().maximumSize(500).build(ColumnVisibility::new); public Attribute() {} @@ -55,12 +54,7 @@ public boolean isMetadataSet() { public ColumnVisibility getColumnVisibility() { if (isMetadataSet()) { Text colVisTxt = metadata.getColumnVisibility(); - if (visibilityCache.containsKey(colVisTxt)) { - return visibilityCache.get(colVisTxt).deepCopy(); - } - ColumnVisibility newColvis = new ColumnVisibility(colVisTxt); - visibilityCache.put(colVisTxt, newColvis); - return newColvis.deepCopy(); + return visibilityCache.get(colVisTxt).deepCopy(); } return Constants.EMPTY_VISIBILITY; } From c210726bd2e680977f5922899463193ac28e9261 Mon Sep 17 00:00:00 2001 From: Joe Alphonso Date: Fri, 17 Jan 2025 10:40:24 -0500 Subject: [PATCH 3/3] use ColumnVisibility copy constructor --- .../src/main/java/datawave/query/attributes/Attribute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java index 32102d6d6e7..e5738cf5b5d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java @@ -54,7 +54,7 @@ public boolean isMetadataSet() { public ColumnVisibility getColumnVisibility() { if (isMetadataSet()) { Text colVisTxt = metadata.getColumnVisibility(); - return visibilityCache.get(colVisTxt).deepCopy(); + return new ColumnVisibility(visibilityCache.get(colVisTxt)); } return Constants.EMPTY_VISIBILITY; }