Skip to content
This repository has been archived by the owner on Dec 5, 2024. It is now read-only.

Simple approach to fix long insert in quontient filter #762

Merged
merged 1 commit into from
Mar 14, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public class QuotientFilter implements Iterable<Long> {
long ELEMENT_MASK;
long MAX_SIZE;
long MAX_INSERTIONS;
int MAX_DUPLICATES = 2;
long[] table;

boolean overflowed = false;
Expand Down Expand Up @@ -357,6 +358,7 @@ public synchronized void insert(byte[] hash) {
}

public synchronized void insert(long hash) {
if (maybeContainsXTimes(hash, MAX_DUPLICATES)) return;
if (entries >= MAX_INSERTIONS | overflowed) {
//Can't safely process an after overflow
//Only a buggy program would attempt it
Expand Down Expand Up @@ -475,6 +477,38 @@ public synchronized boolean maybeContains(long hash) {
return false;
}

public synchronized boolean maybeContainsXTimes(long hash, int num) {
if (overflowed) {
//Can't check for existence after overflow occurred
//and things are missing
throw new OverflowedError();
}

long fq = hashToQuotient(hash);
long fr = hashToRemainder(hash);
long T_fq = getElement(fq);

/* If this quotient has no run, give up. */
if (!isElementOccupied(T_fq)) {
return false;
}

/* Scan the sorted run for the target remainder. */
long s = findRunIndex(fq);
int counter = 0;
do {
long rem = getElementRemainder(getElement(s));
if (rem == fr) {
counter++;
} else if (rem > fr) {
break;
}
s = incrementIndex(s);
}
while (isElementContinuation(getElement(s)));
return counter >= num;
}

/* Remove the entry in QF[s] and slide the rest of the cluster forward. */
void deleteEntry(long s, long quot) {
long next;
Expand Down Expand Up @@ -521,6 +555,7 @@ public void remove(byte[] hash) {
}

public synchronized void remove(long hash) {
if (maybeContainsXTimes(hash, MAX_DUPLICATES)) return;
//Can't safely process a remove after overflow
//Only a buggy program would attempt it
if (overflowed) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
package org.ethereum.db;

import org.ethereum.crypto.HashUtil;
import org.ethereum.datasource.QuotientFilter;
import org.ethereum.util.ByteUtil;
import org.junit.Ignore;
import org.junit.Test;

import static org.ethereum.crypto.HashUtil.sha3;
Expand All @@ -14,7 +11,6 @@
*/
public class QuotientFilterTest {

@Ignore
@Test
public void perfTest() {
QuotientFilter f = QuotientFilter.create(50_000_000, 100_000);
Expand All @@ -35,7 +31,6 @@ public void perfTest() {
}
}

@Ignore
@Test
public void doubleInsertTest() {
QuotientFilter f = QuotientFilter.create(50_000_000, 1000);
Expand Down