Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Colvis cache #2700

Closed
wants to merge 3 commits into from
Closed

Colvis cache #2700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b4c12ba
Attribute ColumnVisibility optimizations
Dec 16, 2024
bbc2390
swap LRUMap for Caffeine cache
Dec 27, 2024
c210726
use ColumnVisibility copy constructor
Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<surefire.forkCount>1C</surefire.forkCount>
<version.accumulo>2.1.2</version.accumulo>
<version.accumulo>2.1.4-SNAPSHOT</version.accumulo>
<version.arquillian>1.4.1.Final</version.arquillian>
<version.arquillian-weld-ee-embedded>1.0.0.Final</version.arquillian-weld-ee-embedded>
<version.assertj>3.20.2</version.assertj>
Expand Down
25 changes: 19 additions & 6 deletions warehouse/query-core/src/main/java/datawave/query/attributes/Attribute.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import com.esotericsoftware.kryo.KryoSerializable;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;

import datawave.query.Constants;
import datawave.query.jexl.DatawaveJexlContext;
Expand All @@ -36,6 +38,7 @@ public abstract class Attribute<T extends Comparable<T>> implements WritableComp
protected Key metadata = null;
protected boolean toKeep = true; // a flag denoting whether this attribute is to be kept in the returned results (transient or not)
protected boolean fromIndex = true; // Assume attributes are from the index unless specified otherwise.
private static final LoadingCache<Text,ColumnVisibility> visibilityCache = Caffeine.newBuilder().maximumSize(500).build(ColumnVisibility::new);

public Attribute() {}

Expand All @@ -50,12 +53,17 @@ public boolean isMetadataSet() {

public ColumnVisibility getColumnVisibility() {
if (isMetadataSet()) {
return metadata.getColumnVisibilityParsed();
Text colVisTxt = metadata.getColumnVisibility();
return new ColumnVisibility(visibilityCache.get(colVisTxt));
}
return Constants.EMPTY_VISIBILITY;
}

public void setColumnVisibility(ColumnVisibility columnVisibility) {
public void setColumnVisibility(ColumnVisibility visibility) {
setColumnVisibility(new Text(visibility.getExpression()));
}

private void setColumnVisibility(Text columnVisibility) {
if (isMetadataSet()) {
metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), metadata.getColumnQualifier(), columnVisibility, metadata.getTimestamp());
} else {
Expand All @@ -82,7 +90,12 @@ public void setTimestamp(long ts) {
*
* Set the metadata. This should only be set here or from extended classes.
*/
protected void setMetadata(ColumnVisibility vis, long ts) {
protected void setMetadata(ColumnVisibility visibility, long ts) {
Text vis = new Text(visibility.getExpression());
setMetadata(vis, ts);
}

private void setMetadata(Text vis, long ts) {
if (isMetadataSet()) {
metadata = new Key(metadata.getRow(), metadata.getColumnFamily(), metadata.getColumnQualifier(), vis, ts);
} else {
Expand Down Expand Up @@ -191,7 +204,7 @@ protected void readMetadata(DataInput in) throws IOException {

in.readFully(cvBytes);

this.setMetadata(new ColumnVisibility(cvBytes), in.readLong());
this.setMetadata(new Text(cvBytes), in.readLong());
} else {
this.clearMetadata();
}
Expand All @@ -201,7 +214,7 @@ protected void readMetadata(Kryo kryo, Input input) {
if (input.readBoolean()) {
int size = input.readInt(true);

this.setMetadata(new ColumnVisibility(input.readBytes(size)), input.readLong());
this.setMetadata(new Text(input.readBytes(size)), input.readLong());
} else {
this.clearMetadata();
}
Expand Down Expand Up @@ -301,7 +314,7 @@ protected static long sizeInBytes(String value) {
if (value == null) {
return 0;
} else {
return 16 + roundUp(12 + (value.length() * 2));
return 16 + roundUp(12 + (value.length() * 2L));
// 16 for int, array ref, and object overhead
// 12 for array overhead
}
Expand Down
Loading