From 7a1a8f5880e18dc11de35b1177b7a4586e625606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Sat, 28 Dec 2024 08:35:47 +0100 Subject: [PATCH 01/13] GH-5221 add test --- .../testsuite/sail/RDFNotifyingStoreTest.java | 163 +++++++++++++++++- 1 file changed, 162 insertions(+), 1 deletion(-) diff --git a/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java b/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java index 9f15fd3d9d0..f6fa783670e 100644 --- a/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java +++ b/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java @@ -13,11 +13,21 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.model.vocabulary.RDF; import org.eclipse.rdf4j.model.vocabulary.RDFS; +import org.eclipse.rdf4j.repository.sail.SailRepository; +import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection; import org.eclipse.rdf4j.sail.NotifyingSail; +import org.eclipse.rdf4j.sail.NotifyingSailConnection; import org.eclipse.rdf4j.sail.SailChangedEvent; import org.eclipse.rdf4j.sail.SailChangedListener; +import org.eclipse.rdf4j.sail.SailConnectionListener; import org.eclipse.rdf4j.sail.SailException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,6 +46,7 @@ public abstract class RDFNotifyingStoreTest extends RDFStoreTest implements Sail private int removeEventCount; private int addEventCount; + private SailRepository repo; /*---------* * Methods * @@ -54,7 +65,9 @@ public abstract class RDFNotifyingStoreTest extends RDFStoreTest implements Sail public void addSailChangedListener() { // set self as listener ((NotifyingSail) sail).addSailChangedListener(this); - + removeEventCount = 0; + addEventCount = 0; + this.repo = new SailRepository(sail); } @Test @@ -99,6 +112,154 @@ public void testNotifyingRemoveAndClear() { assertEquals(3, removeEventCount, "There should have been 3 events in which statements were removed"); } + @Test + public void testUpdateQuery() { + + try (SailRepositoryConnection connection = repo.getConnection()) { + connection.begin(); + connection.add(painter, RDF.TYPE, RDFS.CLASS); + connection.add(painting, RDF.TYPE, RDFS.CLASS); + connection.add(picasso, RDF.TYPE, painter); + connection.add(guernica, RDF.TYPE, painting); + connection.add(picasso, paints, guernica); + connection.commit(); + + } + + addEventCount = 0; + removeEventCount = 0; + + try (SailRepositoryConnection connection = repo.getConnection()) { + Set added = new HashSet<>(); + Set removed = new HashSet<>(); + + List addedRaw = new ArrayList<>(); + List removedRaw = new ArrayList<>(); + + ((NotifyingSailConnection) connection.getSailConnection()) + .addConnectionListener(new SailConnectionListener() { + @Override + public void statementAdded(Statement st) { + boolean add = added.add(st); + if (!add) { + removed.remove(st); + } + + addedRaw.add(st); + } + + @Override + public void statementRemoved(Statement st) { + boolean add = removed.add(st); + if (!add) { + added.remove(st); + } + + removedRaw.add(st); + } + } + ); + + connection.prepareUpdate("" + + "DELETE {?a ?b ?c}" + + "INSERT {?a ?b ?c}" + + "WHERE {?a ?b ?c}").execute(); + + System.out.println("Added Raw Size: " + addedRaw.size()); + System.out.println("Removed Raw Size: " + removedRaw.size()); + System.out.println("Added Raw: " + addedRaw); + System.out.println("Removed Raw: " + removedRaw); + System.out.println("Added Size: " + added.size()); + System.out.println("Removed Size: " + removed.size()); + System.out.println("Added: " + added); + System.out.println("Removed: " + removed); + + assertEquals(5, added.size()); + assertEquals(5, removed.size()); + assertEquals(5, addedRaw.size()); + assertEquals(5, removedRaw.size()); + + assertEquals(added, removed); + + } + + assertEquals(5, con.size()); + + } + + @Test + public void testUpdateQuery2() { + + try (SailRepositoryConnection connection = repo.getConnection()) { + connection.begin(); + connection.add(painter, RDF.TYPE, RDFS.CLASS); + connection.commit(); + + } + + String statement = "<" + painter + "> <" + RDF.TYPE + "> <" + RDFS.CLASS + "> ."; + + addEventCount = 0; + removeEventCount = 0; + + try (SailRepositoryConnection connection = repo.getConnection()) { + Set added = new HashSet<>(); + Set removed = new HashSet<>(); + + List addedRaw = new ArrayList<>(); + List removedRaw = new ArrayList<>(); + + ((NotifyingSailConnection) connection.getSailConnection()) + .addConnectionListener(new SailConnectionListener() { + @Override + public void statementAdded(Statement st) { + boolean add = added.add(st); + if (!add) { + removed.remove(st); + } + + addedRaw.add(st); + } + + @Override + public void statementRemoved(Statement st) { + boolean add = removed.add(st); + if (!add) { + added.remove(st); + } + + removedRaw.add(st); + } + } + ); + + connection.prepareUpdate("" + + "DELETE {" + statement + "}" + + "INSERT {" + statement + "}" + + "WHERE {?a ?b ?c}").execute(); + + System.out.println("Added Raw Size: " + addedRaw.size()); + System.out.println("Removed Raw Size: " + removedRaw.size()); + System.out.println("Added Raw: " + addedRaw); + System.out.println("Removed Raw: " + removedRaw); + System.out.println("Added Size: " + added.size()); + System.out.println("Removed Size: " + removed.size()); + System.out.println("Added: " + added); + System.out.println("Removed: " + removed); + + assertEquals(1, added.size()); + assertEquals(1, removed.size()); + assertEquals(1, addedRaw.size()); + assertEquals(1, removedRaw.size()); + + assertEquals(added, removed); + + } + + assertEquals(1, con.size()); + + } + @Override public void sailChanged(SailChangedEvent event) { if (event.statementsAdded()) { From 28edab6167eb781b9b88267d9be32db4b9f31a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Sat, 28 Dec 2024 08:42:59 +0100 Subject: [PATCH 02/13] GH-5221 fix bug where added statements from a SPARQL update were not notified and were also not added. This only happens when the exact statement was removed by the DELETE clause. --- .../sail/helpers/SailUpdateExecutor.java | 2 +- .../org/eclipse/rdf4j/sail/base/Changeset.java | 4 +++- .../rdf4j/sail/base/SailSourceConnection.java | 13 ++++++++++--- .../valuefactory/ExtensibleStatementImpl.java | 15 +++++++-------- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/core/repository/sail/src/main/java/org/eclipse/rdf4j/repository/sail/helpers/SailUpdateExecutor.java b/core/repository/sail/src/main/java/org/eclipse/rdf4j/repository/sail/helpers/SailUpdateExecutor.java index 06fcf840f25..edd0f0a812c 100644 --- a/core/repository/sail/src/main/java/org/eclipse/rdf4j/repository/sail/helpers/SailUpdateExecutor.java +++ b/core/repository/sail/src/main/java/org/eclipse/rdf4j/repository/sail/helpers/SailUpdateExecutor.java @@ -447,8 +447,8 @@ protected void executeModify(Modify modify, UpdateContext uc, int maxExecutionTi whereClause, uc, maxExecutionTime)) { while (sourceBindings.hasNext()) { BindingSet sourceBinding = sourceBindings.next(); - deleteBoundTriples(sourceBinding, modify.getDeleteExpr(), uc); + deleteBoundTriples(sourceBinding, modify.getDeleteExpr(), uc); insertBoundTriples(sourceBinding, modify.getInsertExpr(), uc); } } diff --git a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/Changeset.java b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/Changeset.java index 9dbeeeaf5ca..2283a3e1c96 100644 --- a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/Changeset.java +++ b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/Changeset.java @@ -29,6 +29,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.eclipse.rdf4j.common.annotation.Experimental; import org.eclipse.rdf4j.common.annotation.InternalUseOnly; import org.eclipse.rdf4j.common.transaction.IsolationLevels; import org.eclipse.rdf4j.model.IRI; @@ -175,7 +176,8 @@ boolean hasApproved(Resource subj, IRI pred, Value obj, Resource[] contexts) { } } - boolean hasDeprecated(Resource subj, IRI pred, Value obj, Resource[] contexts) { + @Experimental + public boolean hasDeprecated(Resource subj, IRI pred, Value obj, Resource[] contexts) { assert !closed; if ((deprecated == null || deprecatedEmpty) && deprecatedContexts == null) { return false; diff --git a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java index 7020c983396..7942984593a 100644 --- a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java +++ b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java @@ -763,8 +763,13 @@ private void add(Resource subj, IRI pred, Value obj, SailDataset dataset, SailSi if (hasConnectionListeners()) { if (!hasStatement(dataset, subj, pred, obj, NULL_CTX)) { notifyStatementAdded(vf.createStatement(subj, pred, obj)); - sink.approve(subj, pred, obj, null); + } else if (sink instanceof Changeset && ((Changeset) sink).hasDeprecated(subj, pred, obj, NULL_CTX)) { + notifyStatementAdded(vf.createStatement(subj, pred, obj)); } + + // always approve the statement, even if it already exists + sink.approve(subj, pred, obj, null); + } else { sink.approve(subj, pred, obj, null); } @@ -784,8 +789,11 @@ private void add(Resource subj, IRI pred, Value obj, SailDataset dataset, SailSi if (hasConnectionListeners()) { if (!hasStatement(dataset, subj, pred, obj, contextsToCheck)) { notifyStatementAdded(vf.createStatement(subj, pred, obj, ctx)); - sink.approve(subj, pred, obj, ctx); + } else if (sink instanceof Changeset + && ((Changeset) sink).hasDeprecated(subj, pred, obj, contextsToCheck)) { + notifyStatementAdded(vf.createStatement(subj, pred, obj)); } + sink.approve(subj, pred, obj, ctx); } else { sink.approve(subj, pred, obj, ctx); } @@ -830,7 +838,6 @@ private boolean remove(Resource subj, IRI pred, Value obj, SailDataset dataset, while (iter.hasNext()) { Statement st = iter.next(); sink.deprecate(st); - statementsRemoved = true; notifyStatementRemoved(st); } diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/valuefactory/ExtensibleStatementImpl.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/valuefactory/ExtensibleStatementImpl.java index b3f8ac04eb7..1e564ac837f 100644 --- a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/valuefactory/ExtensibleStatementImpl.java +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/valuefactory/ExtensibleStatementImpl.java @@ -14,6 +14,7 @@ import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.model.Value; import org.eclipse.rdf4j.model.impl.GenericStatement; @@ -45,19 +46,17 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof ExtensibleStatementImpl)) { + if (!(o instanceof Statement)) { return false; } + if (!(o instanceof ExtensibleStatement)) { + return super.equals(o); + } if (!super.equals(o)) { return false; } - ExtensibleStatementImpl that = (ExtensibleStatementImpl) o; - return inferred == that.inferred; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), inferred); + ExtensibleStatement that = (ExtensibleStatement) o; + return inferred == that.isInferred(); } } From f5aeee30865099b662139d53257e6ea0b1f3444a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Sat, 28 Dec 2024 08:43:26 +0100 Subject: [PATCH 03/13] add a small cache to the regex implementation --- .../RegexValueEvaluationStepSupplier.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/evaluationsteps/RegexValueEvaluationStepSupplier.java b/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/evaluationsteps/RegexValueEvaluationStepSupplier.java index 5fead2eab96..0d2fa5c7b52 100644 --- a/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/evaluationsteps/RegexValueEvaluationStepSupplier.java +++ b/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/evaluationsteps/RegexValueEvaluationStepSupplier.java @@ -38,6 +38,9 @@ public class RegexValueEvaluationStepSupplier { private static final class ChangingRegexQueryValueEvaluationStep implements QueryValueEvaluationStep { private final Regex node; private final EvaluationStrategy strategy; + private Value parg; + private Value farg; + private Pattern pattern; private ChangingRegexQueryValueEvaluationStep(Regex node, EvaluationStrategy strategy) { this.node = node; @@ -56,16 +59,33 @@ public Value evaluate(BindingSet bindings) throws QueryEvaluationException { if (QueryEvaluationUtility.isStringLiteral(arg) && QueryEvaluationUtility.isSimpleLiteral(parg) && (farg == null || QueryEvaluationUtility.isSimpleLiteral(farg))) { + + Pattern pattern = getPattern((Literal) parg, farg); + String text = ((Literal) arg).getLabel(); - String ptn = ((Literal) parg).getLabel(); - // TODO should this Pattern be cached? - int f = extractRegexFlags(farg); - Pattern pattern = Pattern.compile(ptn, f); boolean result = pattern.matcher(text).find(); return BooleanLiteral.valueOf(result); } throw new ValueExprEvaluationException(); } + + private Pattern getPattern(Literal parg, Value farg) { + if (this.parg == parg && this.farg == farg) { + return pattern; + } + + String ptn = parg.getLabel(); + int f = extractRegexFlags(farg); + Pattern pattern = Pattern.compile(ptn, f); + + // cache the pattern object and the current parg and farg so that we can reuse it if the parg and farg are + // reused or somehow constant + this.parg = parg; + this.farg = farg; + this.pattern = pattern; + + return pattern; + } } public static QueryValueEvaluationStep make(EvaluationStrategy strategy, Regex node, From b66f5e9c6d3491326892622572a54ae6afd744ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Sat, 28 Dec 2024 11:11:17 +0100 Subject: [PATCH 04/13] code cleanup --- .../testsuite/sail/RDFNotifyingStoreTest.java | 109 ++++++------------ 1 file changed, 35 insertions(+), 74 deletions(-) diff --git a/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java b/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java index f6fa783670e..85cafbe4c4b 100644 --- a/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java +++ b/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java @@ -126,9 +126,6 @@ public void testUpdateQuery() { } - addEventCount = 0; - removeEventCount = 0; - try (SailRepositoryConnection connection = repo.getConnection()) { Set added = new HashSet<>(); Set removed = new HashSet<>(); @@ -136,44 +133,13 @@ public void testUpdateQuery() { List addedRaw = new ArrayList<>(); List removedRaw = new ArrayList<>(); - ((NotifyingSailConnection) connection.getSailConnection()) - .addConnectionListener(new SailConnectionListener() { - @Override - public void statementAdded(Statement st) { - boolean add = added.add(st); - if (!add) { - removed.remove(st); - } - - addedRaw.add(st); - } - - @Override - public void statementRemoved(Statement st) { - boolean add = removed.add(st); - if (!add) { - added.remove(st); - } - - removedRaw.add(st); - } - } - ); + registerConnectionListener(connection, added, removed, addedRaw, removedRaw); connection.prepareUpdate("" + "DELETE {?a ?b ?c}" + "INSERT {?a ?b ?c}" + "WHERE {?a ?b ?c}").execute(); - System.out.println("Added Raw Size: " + addedRaw.size()); - System.out.println("Removed Raw Size: " + removedRaw.size()); - System.out.println("Added Raw: " + addedRaw); - System.out.println("Removed Raw: " + removedRaw); - System.out.println("Added Size: " + added.size()); - System.out.println("Removed Size: " + removed.size()); - System.out.println("Added: " + added); - System.out.println("Removed: " + removed); - assertEquals(5, added.size()); assertEquals(5, removed.size()); assertEquals(5, addedRaw.size()); @@ -193,15 +159,11 @@ public void testUpdateQuery2() { try (SailRepositoryConnection connection = repo.getConnection()) { connection.begin(); connection.add(painter, RDF.TYPE, RDFS.CLASS); + connection.add(painting, RDF.TYPE, RDFS.CLASS); connection.commit(); } - String statement = "<" + painter + "> <" + RDF.TYPE + "> <" + RDFS.CLASS + "> ."; - - addEventCount = 0; - removeEventCount = 0; - try (SailRepositoryConnection connection = repo.getConnection()) { Set added = new HashSet<>(); Set removed = new HashSet<>(); @@ -209,57 +171,56 @@ public void testUpdateQuery2() { List addedRaw = new ArrayList<>(); List removedRaw = new ArrayList<>(); - ((NotifyingSailConnection) connection.getSailConnection()) - .addConnectionListener(new SailConnectionListener() { - @Override - public void statementAdded(Statement st) { - boolean add = added.add(st); - if (!add) { - removed.remove(st); - } - - addedRaw.add(st); - } - - @Override - public void statementRemoved(Statement st) { - boolean add = removed.add(st); - if (!add) { - added.remove(st); - } + registerConnectionListener(connection, added, removed, addedRaw, removedRaw); - removedRaw.add(st); - } - } - ); + String statement = "<" + painter + "> <" + RDF.TYPE + "> <" + RDFS.CLASS + "> ."; connection.prepareUpdate("" + "DELETE {" + statement + "}" + "INSERT {" + statement + "}" + "WHERE {?a ?b ?c}").execute(); - System.out.println("Added Raw Size: " + addedRaw.size()); - System.out.println("Removed Raw Size: " + removedRaw.size()); - System.out.println("Added Raw: " + addedRaw); - System.out.println("Removed Raw: " + removedRaw); - System.out.println("Added Size: " + added.size()); - System.out.println("Removed Size: " + removed.size()); - System.out.println("Added: " + added); - System.out.println("Removed: " + removed); - assertEquals(1, added.size()); assertEquals(1, removed.size()); - assertEquals(1, addedRaw.size()); - assertEquals(1, removedRaw.size()); + assertEquals(2, addedRaw.size()); + assertEquals(2, removedRaw.size()); assertEquals(added, removed); } - assertEquals(1, con.size()); + assertEquals(2, con.size()); } + private static void registerConnectionListener(SailRepositoryConnection connection, Set added, + Set removed, List addedRaw, List removedRaw) { + ((NotifyingSailConnection) connection.getSailConnection()) + .addConnectionListener( + new SailConnectionListener() { + @Override + public void statementAdded(Statement st) { + boolean add = added.add(st); + if (!add) { + removed.remove(st); + } + + addedRaw.add(st); + } + + @Override + public void statementRemoved(Statement st) { + boolean add = removed.add(st); + if (!add) { + added.remove(st); + } + + removedRaw.add(st); + } + } + ); + } + @Override public void sailChanged(SailChangedEvent event) { if (event.statementsAdded()) { From 557a5f632626d71b97efe40e53a1573b3df0f537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Sat, 28 Dec 2024 12:48:30 +0100 Subject: [PATCH 05/13] fixed bug in Extensible Store --- .../sail/extensiblestore/ReadCommittedWrapper.java | 12 ++++++++---- .../rdf4j/testsuite/sail/RDFNotifyingStoreTest.java | 7 ++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ReadCommittedWrapper.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ReadCommittedWrapper.java index 20ddfd0dc6b..4e211848902 100644 --- a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ReadCommittedWrapper.java +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ReadCommittedWrapper.java @@ -50,14 +50,18 @@ class ReadCommittedWrapper implements DataStructureInterface { @Override public void addStatement(ExtensibleStatement statement) { - internalAdded.put(statement, statement); - internalRemoved.remove(statement); - + ExtensibleStatement put = internalAdded.put(statement, statement); + if (put == null) { + internalRemoved.remove(statement); + } } @Override public void removeStatement(ExtensibleStatement statement) { - internalRemoved.put(statement, statement); + ExtensibleStatement put = internalRemoved.put(statement, statement); + if (put == null) { + internalAdded.remove(statement); + } } diff --git a/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java b/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java index 85cafbe4c4b..53c5265a097 100644 --- a/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java +++ b/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/RDFNotifyingStoreTest.java @@ -180,12 +180,13 @@ public void testUpdateQuery2() { "INSERT {" + statement + "}" + "WHERE {?a ?b ?c}").execute(); - assertEquals(1, added.size()); - assertEquals(1, removed.size()); + assertEquals(added, removed, "Added (expected) is not the same as removed (actual)"); + assertEquals(2, addedRaw.size()); assertEquals(2, removedRaw.size()); - assertEquals(added, removed); + assertEquals(1, added.size()); + assertEquals(1, removed.size()); } From 1db80b5784c566333c917e96e39ea52a46e0299d Mon Sep 17 00:00:00 2001 From: Andreas Schwarte Date: Wed, 15 Jan 2025 07:21:36 +0100 Subject: [PATCH 06/13] GH-5229: fix left bind join in FedX for single binding input This change fixes a situation that can incorrectly cause empty results. It happens when the input of the left argument is a single binding set and for special source selection situations (e.g. the right argument is marked as ExclusiveStatement while the endpoint does not provide data) To avoid the issue we also use the regular left join logic also for a single binding set input, which can handle the situation properly. Issue is covered with a unit test. --- .../evaluation/FederationEvalStrategy.java | 4 - .../rdf4j/federated/BindLeftJoinTests.java | 79 +++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java index 5dafe137315..9a1311d44ca 100644 --- a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java +++ b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java @@ -953,10 +953,6 @@ public abstract CloseableIteration evaluateGroupedCheck( */ public CloseableIteration evaluateLeftBoundJoinStatementPattern( StatementTupleExpr stmt, final List bindings) throws QueryEvaluationException { - // we can omit the bound join handling - if (bindings.size() == 1) { - return evaluate(stmt, bindings.get(0)); - } FilterValueExpr filterExpr = null; if (stmt instanceof FilterTuple) { diff --git a/tools/federation/src/test/java/org/eclipse/rdf4j/federated/BindLeftJoinTests.java b/tools/federation/src/test/java/org/eclipse/rdf4j/federated/BindLeftJoinTests.java index d19359f5bd7..2680c5a8700 100644 --- a/tools/federation/src/test/java/org/eclipse/rdf4j/federated/BindLeftJoinTests.java +++ b/tools/federation/src/test/java/org/eclipse/rdf4j/federated/BindLeftJoinTests.java @@ -14,6 +14,8 @@ import java.util.Set; import org.eclipse.rdf4j.common.iteration.Iterations; +import org.eclipse.rdf4j.federated.endpoint.Endpoint; +import org.eclipse.rdf4j.federated.structures.SubQuery; import org.eclipse.rdf4j.model.util.Values; import org.eclipse.rdf4j.model.vocabulary.FOAF; import org.eclipse.rdf4j.model.vocabulary.OWL; @@ -339,4 +341,81 @@ public void test_leftBindJoin_emptyOptional(boolean bindLeftJoinOptimizationEnab } } + @ParameterizedTest + @ValueSource(booleans = { true, false }) + public void test_leftBindJoin_emptyLeftArgumentAsExclusiveGroup(boolean bindLeftJoinOptimizationEnabled) + throws Exception { + + var endpoints = prepareTest( + Arrays.asList("/tests/basic/data_emptyStore.ttl", "/tests/basic/data_emptyStore.ttl")); + + Repository repo1 = getRepository(1); + Repository repo2 = getRepository(2); + + Repository fedxRepo = fedxRule.getRepository(); + + fedxRule.setConfig(config -> { + config.withBoundJoinBlockSize(10); + config.withEnableOptionalAsBindJoin(bindLeftJoinOptimizationEnabled); + }); + + // add a person + try (RepositoryConnection conn = repo1.getConnection()) { + var p = Values.iri("http://ex.com/p1"); + var otherP = Values.iri("http://other.com/p1"); + conn.add(p, OWL.SAMEAS, otherP); + } + + // add name for person 1 + try (RepositoryConnection conn = repo2.getConnection()) { + var otherP = Values.iri("http://other.com/p1"); + conn.add(otherP, FOAF.NAME, Values.literal("Person 1")); + } + + // mark that repo2 for some reason has foaf:age statements (e.g. old cache entry) + Endpoint repo2Endpoint = endpoints.get(1); + federationContext().getSourceSelectionCache() + .updateInformation(new SubQuery(null, FOAF.AGE, null), repo2Endpoint, true); + + fedxRule.enableDebug(); + + try { + // run query which joins results from multiple repos + // the age does not exist for any person + try (RepositoryConnection conn = fedxRepo.getConnection()) { + String query = "PREFIX foaf: " + + "SELECT * WHERE { " + + " ?person owl:sameAs ?otherPerson . " + + " OPTIONAL { ?otherPerson foaf:age ?age . } " // age does not exist, however is marked as + // ExclusiveStatement + + "}"; + + TupleQuery tupleQuery = conn.prepareTupleQuery(query); + try (TupleQueryResult tqr = tupleQuery.evaluate()) { + var bindings = Iterations.asList(tqr); + + Assertions.assertEquals(1, bindings.size()); + + for (int i = 1; i <= 1; i++) { + var p = Values.iri("http://ex.com/p" + i); + var otherP = Values.iri("http://other.com/p" + i); + + // find the bindingset for the person in the unordered result + BindingSet bs = bindings.stream() + .filter(b -> b.getValue("person").equals(p)) + .findFirst() + .orElseThrow(); + + Assertions.assertEquals(otherP, bs.getValue("otherPerson")); + + Assertions.assertEquals(otherP, bs.getValue("otherPerson")); + Assertions.assertFalse(bs.hasBinding("age")); + } + } + } + + } finally { + fedxRepo.shutDown(); + } + } } From 29097d1492b4f67dd14c0d96b83dd49d6b4055af Mon Sep 17 00:00:00 2001 From: Andreas Schwarte Date: Wed, 15 Jan 2025 13:12:43 +0100 Subject: [PATCH 07/13] GH-5231: fix poor query performance for hasStatements() in FedX The previous implementation of the FedXConnection was delegating "hasStatements()" to the implementation of "getStatements()", where the latter was actually fetching data from the federation members. For checks hasStatements() checks like {null, rdf:type, null} or even {null, null, null} the implementation is problematic as it would fetch all data matching the pattern from the federation members, only to answer if it actually exists. We now make use of "existence" check on the federation members, and can actually rely on the source selection cache for this. Unit test coverage has been added. --- .../rdf4j/federated/FedXConnection.java | 21 +++++++ .../evaluation/FederationEvalStrategy.java | 43 ++++++++++++++- .../eclipse/rdf4j/federated/BasicTests.java | 55 +++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) diff --git a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/FedXConnection.java b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/FedXConnection.java index e1df6b6ca8b..e57e647ae20 100644 --- a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/FedXConnection.java +++ b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/FedXConnection.java @@ -337,6 +337,27 @@ protected SailException convert(RuntimeException e) { } } + @Override + protected boolean hasStatementInternal(Resource subj, IRI pred, Value obj, boolean includeInferred, + Resource[] contexts) { + try { + Dataset dataset = new SimpleDataset(); + FederationEvalStrategy strategy = federationContext.createStrategy(dataset); + QueryInfo queryInfo = new QueryInfo(subj, pred, obj, 0, includeInferred, federationContext, strategy, + dataset); + federationContext.getMonitoringService().monitorQuery(queryInfo); + return strategy.hasStatements(queryInfo, subj, pred, obj, contexts); + + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new SailException(e); + } + } + @Override protected void addStatementInternal(Resource subj, IRI pred, Value obj, Resource... contexts) throws SailException { try { diff --git a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java index 5dafe137315..23d4214cab5 100644 --- a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java +++ b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java @@ -562,7 +562,7 @@ public CloseableIteration getStatements(QueryInfo queryInfo, Resource IRI pred, Value obj, Resource... contexts) throws RepositoryException, MalformedQueryException, QueryEvaluationException { - List members = federationContext.getFederation().getMembers(); + List members = getAccessibleFederationMembers(queryInfo); // a bound query: if at least one fed member provides results // return the statement, otherwise empty result @@ -605,6 +605,47 @@ public CloseableIteration getStatements(QueryInfo queryInfo, Resource return union; } + /** + * Returns true if the federation has statements + * + * @param queryInfo information about the query + * @param subj the subject or null + * @param pred the predicate or null + * @param obj the object or null + * @param contexts optional list of contexts + * @return the statement iteration + * + * @throws RepositoryException + * @throws MalformedQueryException + * @throws QueryEvaluationException + */ + public boolean hasStatements(QueryInfo queryInfo, Resource subj, + IRI pred, Value obj, Resource... contexts) + throws RepositoryException, MalformedQueryException, QueryEvaluationException { + + List members = getAccessibleFederationMembers(queryInfo); + + // form the union of results from relevant endpoints + List sources = CacheUtils.checkCacheForStatementSourcesUpdateCache(cache, members, subj, pred, + obj, queryInfo, contexts); + + if (sources.isEmpty()) { + return false; + } + + return true; + } + + /** + * Returns the accessible federation members in the context of the query. By default this is all federation members. + * + * @param queryInfo + * @return + */ + protected List getAccessibleFederationMembers(QueryInfo queryInfo) { + return federationContext.getFederation().getMembers(); + } + public CloseableIteration evaluateService(FedXService service, BindingSet bindings) throws QueryEvaluationException { diff --git a/tools/federation/src/test/java/org/eclipse/rdf4j/federated/BasicTests.java b/tools/federation/src/test/java/org/eclipse/rdf4j/federated/BasicTests.java index 962ded27832..51566c1cb47 100644 --- a/tools/federation/src/test/java/org/eclipse/rdf4j/federated/BasicTests.java +++ b/tools/federation/src/test/java/org/eclipse/rdf4j/federated/BasicTests.java @@ -139,6 +139,61 @@ public void testBindClause() throws Exception { execute("/tests/basic/query_bind.rq", "/tests/basic/query_bind.srx", false, true); } + @Test + public void testRepositoryConnectionApi() throws Exception { + + prepareTest( + Arrays.asList("/tests/basic/data_emptyStore.ttl", "/tests/basic/data_emptyStore.ttl")); + + Repository repo1 = getRepository(1); + Repository repo2 = getRepository(2); + + IRI bob = Values.iri("http://example.org/bob"); + IRI alice = Values.iri("http://example.org/alice"); + IRI graph1 = Values.iri("http://example.org/graph1"); + IRI graph2 = Values.iri("http://example.org/graph2"); + + try (RepositoryConnection conn = repo1.getConnection()) { + conn.add(bob, RDF.TYPE, FOAF.PERSON, graph1); + conn.add(bob, FOAF.NAME, Values.literal("Bob"), graph1); + } + + try (RepositoryConnection conn = repo2.getConnection()) { + conn.add(alice, RDF.TYPE, FOAF.PERSON, graph2); + conn.add(alice, FOAF.NAME, Values.literal("Alice"), graph2); + } + + var fedxRepo = fedxRule.getRepository(); + + try (var conn = fedxRepo.getConnection()) { + + // hasStatement which exist + Assertions.assertTrue(conn.hasStatement(bob, RDF.TYPE, FOAF.PERSON, false)); + Assertions.assertTrue(conn.hasStatement(bob, RDF.TYPE, FOAF.PERSON, false, graph1)); + Assertions.assertTrue(conn.hasStatement(null, RDF.TYPE, FOAF.PERSON, false)); + Assertions.assertTrue(conn.hasStatement(null, RDF.TYPE, FOAF.PERSON, false, graph1)); + Assertions.assertTrue(conn.hasStatement(null, RDF.TYPE, null, false)); + Assertions.assertTrue(conn.hasStatement(null, RDF.TYPE, null, false, graph1)); + Assertions.assertTrue(conn.hasStatement(null, RDF.TYPE, null, false, graph2)); + Assertions.assertTrue(conn.hasStatement(null, null, null, false)); + Assertions.assertTrue(conn.hasStatement(null, null, null, false, graph1)); + + // hasStatement which do not exist + Assertions.assertFalse(conn.hasStatement(bob, RDF.TYPE, FOAF.ORGANIZATION, false)); + Assertions.assertFalse(conn.hasStatement(bob, RDF.TYPE, FOAF.PERSON, false, graph2)); + + // getStatements + Assertions.assertEquals(Set.of(bob, alice), + QueryResults.asModel(conn.getStatements(null, RDF.TYPE, FOAF.PERSON, false)).subjects()); + Assertions.assertEquals(Set.of(bob), + QueryResults.asModel(conn.getStatements(null, RDF.TYPE, FOAF.PERSON, false, graph1)).subjects()); + Assertions.assertEquals(Set.of(bob, alice), + QueryResults.asModel(conn.getStatements(null, null, null, false)).subjects()); + Assertions.assertEquals(Set.of(bob), + QueryResults.asModel(conn.getStatements(null, null, null, false, graph1)).subjects()); + } + } + @Test public void testFederationSubSetQuery() throws Exception { String ns1 = "http://namespace1.org/"; From a22d277d38cb429f0079dad2d655f8c8c9bdfcc2 Mon Sep 17 00:00:00 2001 From: Andreas Schwarte Date: Mon, 20 Jan 2025 11:58:30 +0100 Subject: [PATCH 08/13] GH-5234: fix limited config support of FedX source selection cache Add an additional constructor accepting a supplier for an initialized cache. --- .../federated/cache/SourceSelectionMemoryCache.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/cache/SourceSelectionMemoryCache.java b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/cache/SourceSelectionMemoryCache.java index 9457c6f9273..9ef97959c39 100644 --- a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/cache/SourceSelectionMemoryCache.java +++ b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/cache/SourceSelectionMemoryCache.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; import org.eclipse.rdf4j.federated.endpoint.Endpoint; import org.eclipse.rdf4j.federated.exception.FedXRuntimeException; @@ -49,6 +50,14 @@ public SourceSelectionMemoryCache(String cacheSpec) { this.cache = CacheBuilder.from(CacheBuilderSpec.parse(cacheSpec)).build(); } + /** + * + * @param cacheSupplier provider for an instantiated Guava cache + */ + public SourceSelectionMemoryCache(Supplier> cacheSupplier) { + this.cache = cacheSupplier.get(); + } + @Override public StatementSourceAssurance getAssurance(SubQuery subQuery, Endpoint endpoint) { From 99d7ba398700efaf60f17934956d916e91bb1f10 Mon Sep 17 00:00:00 2001 From: Andreas Schwarte Date: Tue, 14 Jan 2025 12:10:52 +0100 Subject: [PATCH 09/13] GH-5227: fix binding assigner optimizer in FedX The federation optimizer was missing to execute the binding assigner (which injects external bindings into the statement pattern). The consequence was potentially incorrect results (due to source source selection with partial knowledge) as well as sub-optimal source selection Issue is covered with a unit test, which is failing in two places prior to this change. --- .../evaluation/FederationEvalStrategy.java | 16 ++++- .../FederationEvalStrategyTest.java | 60 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java index 5dafe137315..2fec83ab3b3 100644 --- a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java +++ b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java @@ -107,6 +107,7 @@ import org.eclipse.rdf4j.query.algebra.ValueExpr; import org.eclipse.rdf4j.query.algebra.Var; import org.eclipse.rdf4j.query.algebra.evaluation.QueryEvaluationStep; +import org.eclipse.rdf4j.query.algebra.evaluation.QueryOptimizer; import org.eclipse.rdf4j.query.algebra.evaluation.QueryValueEvaluationStep; import org.eclipse.rdf4j.query.algebra.evaluation.ValueExprEvaluationException; import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedService; @@ -118,6 +119,7 @@ import org.eclipse.rdf4j.query.algebra.evaluation.iterator.HashJoinIteration; import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.ConstantOptimizer; import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.DisjunctiveConstraintOptimizer; +import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.StandardQueryOptimizerPipeline; import org.eclipse.rdf4j.query.algebra.evaluation.util.QueryEvaluationUtil; import org.eclipse.rdf4j.query.algebra.helpers.TupleExprs; import org.eclipse.rdf4j.query.algebra.helpers.collectors.VarNameCollector; @@ -147,6 +149,14 @@ public abstract class FederationEvalStrategy extends StrictEvaluationStrategy { protected FederationContext federationContext; + /** + * List of standard {@link QueryOptimizer}s applicable to federation + */ + private static final List standardOptimizers = List.of( + StandardQueryOptimizerPipeline.BINDING_ASSIGNER, + StandardQueryOptimizerPipeline.BINDING_SET_ASSIGNMENT_INLINER, + StandardQueryOptimizerPipeline.DISJUNCTIVE_CONSTRAINT_OPTIMIZER); + public FederationEvalStrategy(FederationContext federationContext) { super(new org.eclipse.rdf4j.query.algebra.evaluation.TripleSource() { @@ -209,9 +219,11 @@ public TupleExpr optimize(TupleExpr expr, EvaluationStatistics evaluationStatist } /* original RDF4J optimizers */ - new ConstantOptimizer(this).optimize(query, dataset, bindings); // maybe remove this optimizer later + for (QueryOptimizer optimizer : standardOptimizers) { + optimizer.optimize(query, dataset, bindings); + } - new DisjunctiveConstraintOptimizer().optimize(query, dataset, bindings); + new ConstantOptimizer(this).optimize(query, dataset, bindings); // maybe remove this optimizer later /* * TODO add some generic optimizers: - FILTER ?s=1 && ?s=2 => EmptyResult - Remove variables that are not diff --git a/tools/federation/src/test/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategyTest.java b/tools/federation/src/test/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategyTest.java index 7277b82f3ec..edf43cf5c54 100644 --- a/tools/federation/src/test/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategyTest.java +++ b/tools/federation/src/test/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategyTest.java @@ -10,9 +10,23 @@ *******************************************************************************/ package org.eclipse.rdf4j.federated.evaluation; +import java.util.Arrays; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.eclipse.rdf4j.federated.SPARQLBaseTest; +import org.eclipse.rdf4j.federated.cache.SourceSelectionCache; +import org.eclipse.rdf4j.federated.cache.SourceSelectionCache.StatementSourceAssurance; +import org.eclipse.rdf4j.federated.endpoint.Endpoint; +import org.eclipse.rdf4j.federated.structures.SubQuery; +import org.eclipse.rdf4j.model.util.Values; +import org.eclipse.rdf4j.model.vocabulary.FOAF; +import org.eclipse.rdf4j.model.vocabulary.OWL; +import org.eclipse.rdf4j.model.vocabulary.RDF; +import org.eclipse.rdf4j.query.TupleQuery; +import org.eclipse.rdf4j.repository.Repository; +import org.eclipse.rdf4j.repository.RepositoryConnection; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -46,4 +60,50 @@ public void testOptimize_SingleMember_Service() throws Exception { Assertions.assertTrue(queryPlan.startsWith("QueryRoot")); } + + @Test + public void testSourceSelectionCache_setBindings() throws Exception { + + var bob = Values.iri("http://example.com/bob"); + + List endpoints = prepareTest( + Arrays.asList("/tests/basic/data_emptyStore.ttl", "/tests/basic/data_emptyStore.ttl")); + + Repository repo1 = getRepository(1); + Repository repo2 = getRepository(2); + + String repo1Id = endpoints.get(0).getId(); + + try (RepositoryConnection con = repo1.getConnection()) { + con.add(bob, RDF.TYPE, FOAF.PERSON); + } + + try (RepositoryConnection con = repo2.getConnection()) { + con.add(FOAF.PERSON, RDF.TYPE, OWL.CLASS); + } + + Repository fedxRepo = fedxRule.getRepository(); + + fedxRule.enableDebug(); + + try (var conn = fedxRepo.getConnection()) { + + TupleQuery tq = conn.prepareTupleQuery("SELECT * WHERE { ?s a ?type }"); + tq.setBinding("s", bob); + + try (var tqr = tq.evaluate()) { + // just consume the result + Assertions.assertEquals(Set.of(FOAF.PERSON), + tqr.stream().map(bs -> bs.getValue("type")).collect(Collectors.toSet())); + } + } + + SourceSelectionCache cache = federationContext().getSourceSelectionCache(); + + var assurance = cache.getAssurance(new SubQuery(bob, RDF.TYPE, null), + federationContext().getEndpointManager().getEndpoint(repo1Id)); + + // we expect that the source selection cache can assure statements + Assertions.assertEquals(StatementSourceAssurance.HAS_REMOTE_STATEMENTS, assurance); + } } From 1cc4ab81f6ae14e22fe39de70475206c0a201ba8 Mon Sep 17 00:00:00 2001 From: Andreas Schwarte Date: Thu, 23 Jan 2025 07:25:30 +0100 Subject: [PATCH 10/13] GH-5231: refine javadoc, add Experimental annotation --- .../rdf4j/federated/evaluation/FederationEvalStrategy.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java index 23d4214cab5..27e15607d60 100644 --- a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java +++ b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/evaluation/FederationEvalStrategy.java @@ -17,6 +17,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import org.eclipse.rdf4j.common.annotation.Experimental; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.iteration.EmptyIteration; import org.eclipse.rdf4j.common.iteration.SingletonIteration; @@ -638,10 +639,16 @@ public boolean hasStatements(QueryInfo queryInfo, Resource subj, /** * Returns the accessible federation members in the context of the query. By default this is all federation members. + *

+ * Specialized implementations of the {@link FederationEvalStrategy} may override and define custom behavior (e.g., + * to support resilience). + *

+ * * * @param queryInfo * @return */ + @Experimental protected List getAccessibleFederationMembers(QueryInfo queryInfo) { return federationContext.getFederation().getMembers(); } From 5ceb5fcf3215c7d869b43a1c09389ebab344f9db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20M=2E=20Ottestad?= Date: Sat, 25 Jan 2025 09:26:54 +0100 Subject: [PATCH 11/13] Setup github pages using hugo --- .github/workflows/hugo.yml | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/hugo.yml diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml new file mode 100644 index 00000000000..78cd10c4988 --- /dev/null +++ b/.github/workflows/hugo.yml @@ -0,0 +1,74 @@ +# Sample workflow for building and deploying a Hugo site to GitHub Pages +name: Deploy Hugo site to Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +# Default to bash +defaults: + run: + shell: bash + +jobs: + # Build job + build: + runs-on: ubuntu-latest + env: + HUGO_VERSION: 0.128.0 + steps: + - name: Install Hugo CLI + run: | + wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ + && sudo dpkg -i ${{ runner.temp }}/hugo.deb + - name: Install Dart Sass + run: sudo snap install dart-sass + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Setup Pages + id: pages + uses: actions/configure-pages@v5 + - name: Install Node.js dependencies + run: "[[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true" + - name: Build with Hugo + env: + HUGO_CACHEDIR: ${{ runner.temp }}/hugo_cache + HUGO_ENVIRONMENT: production + run: | + hugo --source site \ + --minify \ + --baseURL "${{ steps.pages.outputs.base_url }}/" + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./site/public + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 From ab8db42930421cefce0dab7a52d39435f04db7cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20M=2E=20Ottestad?= Date: Sat, 25 Jan 2025 09:28:16 +0100 Subject: [PATCH 12/13] GH-5214 lmdb supports linux ppc, updated to latest version and also some cleanup in the bom pom --- bom/pom.xml | 38 +++++++++++++++++++++++++++++++++++++- core/sail/lmdb/pom.xml | 14 ++++++++++++++ pom.xml | 2 +- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 2ce91988ace..adab1ca0dc7 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -413,7 +413,13 @@ org.lwjgl lwjgl-lmdb - natives-macos-arm64 + natives-linux-arm64 + ${lwjgl.version} + + + org.lwjgl + lwjgl-lmdb + natives-linux-ppc64le ${lwjgl.version} @@ -422,18 +428,42 @@ natives-macos ${lwjgl.version} + + org.lwjgl + lwjgl-lmdb + natives-macos-arm64 + ${lwjgl.version} + org.lwjgl lwjgl-lmdb natives-windows ${lwjgl.version} + + org.lwjgl + lwjgl-lmdb + natives-windows-arm64 + ${lwjgl.version} + org.lwjgl lwjgl natives-linux ${lwjgl.version} + + org.lwjgl + lwjgl + natives-linux-arm64 + ${lwjgl.version} + + + org.lwjgl + lwjgl + natives-linux-ppc64le + ${lwjgl.version} + org.lwjgl lwjgl @@ -452,6 +482,12 @@ natives-windows ${lwjgl.version} + + org.lwjgl + lwjgl + natives-windows-arm64 + ${lwjgl.version} + diff --git a/core/sail/lmdb/pom.xml b/core/sail/lmdb/pom.xml index c6afed34356..9547eb72145 100644 --- a/core/sail/lmdb/pom.xml +++ b/core/sail/lmdb/pom.xml @@ -29,6 +29,13 @@ ${lwjgl.version} runtime + + org.lwjgl + lwjgl-lmdb + natives-linux-ppc64le + ${lwjgl.version} + runtime + org.lwjgl lwjgl-lmdb @@ -71,6 +78,13 @@ ${lwjgl.version} runtime + + org.lwjgl + lwjgl + natives-linux-ppc64le + ${lwjgl.version} + runtime + org.lwjgl lwjgl diff --git a/pom.xml b/pom.xml index e72a8ef60f8..cbfb71474bf 100644 --- a/pom.xml +++ b/pom.xml @@ -367,7 +367,7 @@ 0.13.4 5.0.0 2.3.8 - 3.3.3 + 3.3.6 8.9.0 8.9.0 7.15.2 From b0f8edb6b17e8955489b774ff80dc198499ce0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Sat, 25 Jan 2025 09:53:15 +0100 Subject: [PATCH 13/13] wip --- site/layouts/partials/head_suffix.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/site/layouts/partials/head_suffix.html b/site/layouts/partials/head_suffix.html index 9c64e27ed69..82bed5273b8 100644 --- a/site/layouts/partials/head_suffix.html +++ b/site/layouts/partials/head_suffix.html @@ -1,4 +1,4 @@ -