Skip to content

Commit

Permalink
Enh 35900901 - [35884452->24.03] Improve filter re-ordering algorithm…
Browse files Browse the repository at this point in the history 
… for composite filters (re-enable)

[git-p4: depot-paths = "//dev/coherence-ce/release/coherence-ce-v24.03/": change = 107711]
  • Loading branch information
@aseovic
aseovic committed Mar 20, 2024
1 parent 5150be3 commit 501157e
Show file tree
Hide file tree
Showing 7 changed files with 40 additions and 53 deletions.
4 changes: 2 additions & 2 deletions prj/coherence-core/src/main/java/com/tangosol/util/filter/AllFilter.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
*
* Licensed under the Universal Permissive License v 1.0 as shown at
* https://oss.oracle.com/licenses/upl.
Expand Down Expand Up @@ -137,7 +137,7 @@ protected Set<Filter<?>> simplifyFilters()
protected Filter applyIndex(Map mapIndexes, Set setKeys,
QueryContext ctx, QueryRecord.PartialResult.TraceStep step)
{
//optimizeFilterOrder(mapIndexes, setKeys);
optimizeFilterOrder(mapIndexes, setKeys);

Filter[] aFilter = m_aFilter;
int cFilters = aFilter.length;
Expand Down
4 changes: 2 additions & 2 deletions prj/coherence-core/src/main/java/com/tangosol/util/filter/AnyFilter.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
*
* Licensed under the Universal Permissive License v 1.0 as shown at
* https://oss.oracle.com/licenses/upl.
Expand Down Expand Up @@ -149,7 +149,7 @@ protected Set<Filter<?>> simplifyFilters()
protected Filter applyIndex(Map mapIndexes, Set setKeys, QueryContext ctx,
QueryRecord.PartialResult.TraceStep step)
{
//optimizeFilterOrder(mapIndexes, setKeys);
optimizeFilterOrder(mapIndexes, setKeys);

Filter[] aFilter = m_aFilter;
int cFilters = aFilter.length;
Expand Down
5 changes: 2 additions & 3 deletions prj/coherence-core/src/main/java/com/tangosol/util/filter/ArrayFilter.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
*
* Licensed under the Universal Permissive License v 1.0 as shown at
* https://oss.oracle.com/licenses/upl.
Expand All @@ -21,7 +21,6 @@
import com.tangosol.util.QueryContext;

import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;

Expand Down Expand Up @@ -254,7 +253,7 @@ protected void optimizeFilterOrder(Map mapIndexes, Set setKeys)
return;
}

Set<Filter> setFilter = new LinkedHashSet<>(Arrays.asList(m_aFilter));
Set<Filter<?>> setFilter = simplifyFilters();
int cFilters = setFilter.size();
WeightedFilter[] aWeighted = new WeightedFilter[cFilters];
Filter<?>[] aFilter = new Filter[cFilters];
Expand Down
17 changes: 9 additions & 8 deletions prj/test/functional/filter/src/main/java/filter/AbstractFilterTests.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
*
* Licensed under the Universal Permissive License v 1.0 as shown at
* https://oss.oracle.com/licenses/upl.
Expand Down Expand Up @@ -76,8 +76,6 @@
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import org.junit.Test;

Expand Down Expand Up @@ -297,7 +295,7 @@ protected void doTest(NamedCache cacheTest, int cItems,
}

// 2) fill the cache and just execute all the queries
if (!loadData(cacheTest, "people-" + cItems + ".bin"))
if (!loadData(cacheTest, "people-" + cItems + ".bin", fDebug))
{
Person.fillRandom(cacheTest, cItems);
saveData(cacheTest, "people-" + cItems + ".bin");
Expand Down Expand Up @@ -388,9 +386,9 @@ protected void doTest(NamedCache cacheTest, int cItems,
}
}

// the default is 4, which will test 64 random combinations of filters
// the default is 10, which will test 1,000 random combinations of filters
// specify 0 to run all possible combinations (all 64 thousand of them...)
int cIterations = Config.getInteger("test.filters.iterations", 4);
int cIterations = Config.getInteger("test.filters.iterations", 10);
boolean fRandom = true;

if (cIterations == 0)
Expand Down Expand Up @@ -583,15 +581,18 @@ private void saveData(NamedCache cache, String sFileName)
}
}

private boolean loadData(NamedCache cache, String sFileName)
private boolean loadData(NamedCache cache, String sFileName, boolean fDebug)
{
try (FileInputStream inFile = new FileInputStream(sFileName))
{
Map data = new HashMap(cache);
DataInput in = new DataInputStream(inFile);
ExternalizableHelper.readMap(in, data, getClass().getClassLoader());
cache.putAll(data);
System.out.printf("\nLoaded %d cache entries from %s", cache.size(), sFileName);
if (fDebug)
{
System.out.printf("Loaded %d cache entries from %s\n", cache.size(), sFileName);
}
return true;
}
catch (IOException e)
Expand Down
11 changes: 9 additions & 2 deletions prj/test/functional/filter/src/main/java/filter/QueryRecordReporterTests.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
*
* Licensed under the Universal Permissive License v 1.0 as shown at
* https://oss.oracle.com/licenses/upl.
Expand All @@ -10,15 +10,18 @@

import com.oracle.bedrock.testsupport.deferred.Eventually;
import com.oracle.bedrock.runtime.coherence.CoherenceCacheServer;

import com.tangosol.net.CacheFactory;
import com.tangosol.net.DistributedCacheService;
import com.tangosol.net.NamedCache;
import com.tangosol.net.partition.PartitionSet;

import com.tangosol.util.Filter;
import com.tangosol.util.Resources;
import com.tangosol.util.SimpleQueryRecord;
import com.tangosol.util.SimpleQueryRecordReporter;
import com.tangosol.util.aggregator.QueryRecorder;

import com.tangosol.util.filter.AllFilter;
import com.tangosol.util.filter.AlwaysFilter;
import com.tangosol.util.filter.AndFilter;
Expand All @@ -27,12 +30,14 @@
import com.tangosol.util.filter.LessEqualsFilter;
import com.tangosol.util.filter.LikeFilter;
import com.tangosol.util.filter.PartitionedFilter;

import com.oracle.coherence.testing.AbstractFunctionalTest;

import data.pof.Address;

import filter.nestinglevel1.nestinglevel2.nestinglevel3.IntegerToStringPersonKeyExtractor;
import filter.nestinglevel1.nestinglevel2.nestinglevel3.StringToIntegerPersonAddressZipExtractor;

import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
Expand All @@ -43,7 +48,9 @@
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;

import java.net.URL;

import java.util.Calendar;
import java.util.NoSuchElementException;
import java.util.Scanner;
Expand Down Expand Up @@ -145,7 +152,7 @@ public void testReporterDoubleIndex() throws Exception

out(" * * * * * * * Double-Index Query * * * * * * * * * * * * * * (testReporterDoubleIndex)");
testReporter(cache, filterAll, QueryRecorder.RecordType.EXPLAIN, "explain-DoubleIndex.ptn");
//testReporter(cache, filterAll, QueryRecorder.RecordType.TRACE, "trace-DoubleIndex.ptn");
testReporter(cache, filterAll, QueryRecorder.RecordType.TRACE, "trace-DoubleIndex.ptn");

cache.removeIndex(IntegerToStringPersonKeyExtractor.INSTANCE);
cache.removeIndex(StringToIntegerPersonAddressZipExtractor.INSTANCE);
Expand Down
13 changes: 6 additions & 7 deletions prj/test/functional/filter/src/main/resources/explain-DoubleIndex.ptn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,20 @@
Explain Plan
Filter Name Index Cost
======================================================================================
AllFilter\s+\| ---- \| 0
AllFilter \| ---- \| 0
LikeFilter\(filter\.nestinglevel1\.nestinglevel2\.nestinglev\.\.\. \(0\) \| 0 \| 11
AndFilter\s+\| ---- \| 0
BetweenFilter\(filter\.nestinglevel1\.nestinglevel2\.nesti\.\.\. \(1\) \| 1 \| (66|67)
EqualsFilter\(\.getFirstName\(\), Hector\)\s+\| 2 \| 100000
LessEqualsFilter\(\.getAge\(\), 98\)\s+\| 3 \| 100000
BetweenFilter\(filter\.nestinglevel1\.nestinglevel2\.nesting\.\.\. \(1\) \| 1 \| (66|67)
LessEqualsFilter\(\.getAge\(\), 98\) \| 2 \| 100000
EqualsFilter\(\.getFirstName\(\), Hector\) \| 3 \| 100000


Index Lookups
Index Description Extractor Ordered
======================================================================================
0 \| Partitioned: Footprint=[0-9]+[\.[0-9]+]*[K\|M]*B,\.\.\. \(2\)\s+\| filter\.nestinglevel1\.nes\.\.\. \(3\) \| true
1 \| Partitioned: Footprint=[0-9]+[\.[0-9]+]*[K\|M]*B, Size=3\s+\| filter\.nestinglevel1\.nes\.\.\. \(4\) \| true
2 \| No index found \| \.getFirstName\(\) \| false
3 \| No index found \| \.getAge\(\) \| false
2 \| No index found \| \.getAge\(\) \| false
3 \| No index found \| \.getFirstName\(\) \| false


Complete filter and index descriptions
Expand Down
39 changes: 10 additions & 29 deletions prj/test/functional/filter/src/main/resources/trace-DoubleIndex.ptn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,42 +2,23 @@
Trace
Filter Name Index Effectiveness Duration
======================================================================================
AllFilter[\s]*\| ---- \| 50\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
LikeFilter\(filter\.nestinglevel1\.\.\.\. \(0\) \| 0 \| 50\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
AndFilter[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
BetweenFilter\(filter\.nestingle\.\.\. \(1\) \| 1 \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
EqualsFilter\(\.getFirstName\(\), Hector\) \| 2 \| [0-9]+\|[0-9]+\(0%\)[\s]*\| [0-9]+[\s]*
LessEqualsFilter\(\.getAge\(\), 98\)[\s]*\| 3 \| [0-9]+\|[0-9]+\(0%\)[\s]*\| [0-9]+[\s]*
AllFilter[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
EqualsFilter\(\.getFirstName\(\), Hector\)[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
LessEqualsFilter\(\.getAge\(\), 98\)[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*

PartitionSet\{[0-9]+\.\.[0-9]+\}


Trace
Filter Name Index Effectiveness Duration
======================================================================================
AllFilter[\s]*\| ---- \| 50\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
LikeFilter\(filter\.nestinglevel1\.\.\.\. \(0\) \| 0 \| 50\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
AndFilter[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
BetweenFilter\(filter\.nestingle\.\.\. \(1\) \| 1 \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
EqualsFilter\(\.getFirstName\(\), Hector\) \| 2 \| [0-9]+\|[0-9]+\(0%\)[\s]*\| [0-9]+[\s]*
LessEqualsFilter\(\.getAge\(\), 98\)[\s]*\| 3 \| [0-9]+\|[0-9]+\(0%\)[\s]*\| [0-9]+[\s]*
AllFilter[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
EqualsFilter\(\.getFirstName\(\), Hector\)[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
LessEqualsFilter\(\.getAge\(\), 98\)[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*

PartitionSet\{[0-9]+\.\.[0-9]+\}
AllFilter \| ---- \| 100\|7\(93%\) \| [0-9]+[\s]*
LikeFilter\(filter\.nestinglevel1\.\.\.\. \(0\) \| 0 \| 100\|11\(89%\) \| [0-9]+[\s]*
BetweenFilter\(filter\.nestingleve\.\.\. \(1\) \| 1 \| 11\|7\(36%\) \| [0-9]+[\s]*
LessEqualsFilter\(\.getAge\(\), 98\) \| 2 \| 7\|7\(0%\) \| [0-9]+[\s]*
EqualsFilter\(\.getFirstName\(\), Hector\) \| 3 \| 7\|7\(0%\) \| [0-9]+[\s]*
AllFilter \| ---- \| 7\|1\(85%\) \| [0-9]+[\s]*
LessEqualsFilter\(\.getAge\(\), 98\) \| ---- \| 7\|7\(0%\) \| [0-9]+[\s]*
EqualsFilter\(\.getFirstName\(\), Hector\) \| ---- \| 7\|1\(85%\) \| [0-9]+[\s]*


Index Lookups
Index Description Extractor Ordered
======================================================================================
0 \| Partitioned: Footprint=[0-9]+[\.[0-9]+]*[K\|M]*B,\.\.\. \(2\)\s+\| filter\.nestinglevel1\.nes\.\.\. \(3\) \| true
1 \| Partitioned: Footprint=[0-9]+[\.[0-9]+]*[K\|M]*B, Size=3 \| filter\.nestinglevel1\.nes\.\.\. \(4\) \| true
2 \| No index found \| \.getFirstName\(\) \| false
3 \| No index found \| \.getAge\(\) \| false
2 \| No index found \| \.getAge\(\) \| false
3 \| No index found \| \.getFirstName\(\) \| false


Complete filter and index descriptions
Expand Down

0 comments on commit 501157e

Please sign in to comment.