eclipse-rdf4j/rdf4j#1375 added large benchmark and adjusted the compl…

…ex benchmark shacl rules Signed-off-by: Håvard Ottestad <[email protected]>
eclipse-rdf4j · Apr 1, 2019 · a76c261 · a76c261
1 parent 398da1e
commit a76c261
Show file tree

Hide file tree

Showing 3 changed files with 928,533 additions and 24 deletions.
diff --git a/shacl/src/test/java/org/eclipse/rdf4j/sail/shacl/benchmark/ComplexLargeBenchmark.java b/shacl/src/test/java/org/eclipse/rdf4j/sail/shacl/benchmark/ComplexLargeBenchmark.java
@@ -0,0 +1,234 @@
+/*******************************************************************************
+ * Copyright (c) 2019 Eclipse RDF4J contributors.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *******************************************************************************/
+
+package org.eclipse.rdf4j.sail.shacl.benchmark;
+
+import ch.qos.logback.classic.Logger;
+import org.apache.commons.io.IOUtils;
+import org.eclipse.rdf4j.IsolationLevels;
+import org.eclipse.rdf4j.model.vocabulary.RDFS;
+import org.eclipse.rdf4j.repository.sail.SailRepository;
+import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
+import org.eclipse.rdf4j.rio.RDFFormat;
+import org.eclipse.rdf4j.sail.shacl.ShaclSail;
+import org.eclipse.rdf4j.sail.shacl.ShaclSailConnection;
+import org.eclipse.rdf4j.sail.shacl.Utils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author Håvard Ottestad
+ */
+@State(Scope.Benchmark)
+@Warmup(iterations = 20)
+@BenchmarkMode({Mode.AverageTime})
+@Fork(value = 1, jvmArgs = {"-Xms8G", "-Xmx8G"})
+//@Fork(value = 1, jvmArgs = {"-Xms8G", "-Xmx8G", "-XX:+UnlockCommercialFeatures", "-XX:StartFlightRecording=delay=15s,duration=120s,filename=recording.jfr,settings=ProfilingAggressive.jfc", "-XX:FlightRecorderOptions=samplethreads=true,stackdepth=1024", "-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints"})
+@Measurement(iterations = 10)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+public class ComplexLargeBenchmark {
+
+	private static String transaction1;
+	private static String transaction2;
+
+	static {
+		try {
+			transaction1 = IOUtils.toString(
+				ComplexLargeBenchmark.class.getClassLoader().getResourceAsStream("complexBenchmark/transaction1.qr"),
+				"utf-8");
+			transaction2 = IOUtils.toString(
+				ComplexLargeBenchmark.class.getClassLoader().getResourceAsStream("complexBenchmark/transaction2.qr"),
+				"utf-8");
+
+		} catch (IOException e) {
+			throw new RuntimeException();
+		}
+	}
+
+	SailRepository repository;
+
+	@Setup(Level.Invocation)
+	public void setUp() {
+
+		((Logger) LoggerFactory.getLogger(ShaclSailConnection.class.getName()))
+			.setLevel(ch.qos.logback.classic.Level.ERROR);
+		((Logger) LoggerFactory.getLogger(ShaclSail.class.getName())).setLevel(ch.qos.logback.classic.Level.ERROR);
+
+		try {
+			repository = new SailRepository(Utils.getInitializedShaclSail("complexBenchmark/shacl.ttl"));
+
+			((ShaclSail) repository.getSail()).disableValidation();
+
+			try (SailRepositoryConnection connection = repository.getConnection()) {
+				connection.begin(IsolationLevels.NONE);
+				try (InputStream resourceAsStream = new BufferedInputStream(ComplexLargeBenchmark.class.getClassLoader().getResourceAsStream("complexBenchmark/datagovbe-valid.ttl"))) {
+					connection.add(resourceAsStream, "", RDFFormat.TURTLE);
+				}
+				connection.commit();
+			}
+
+			((ShaclSail) repository.getSail()).enableValidation();
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+
+		System.gc();
+	}
+
+	@TearDown
+	public void teardown() {
+		if (repository != null) {
+			repository.shutDown();
+		}
+	}
+
+	@Benchmark
+	public void shaclParallelCachePreloaded() {
+
+		((ShaclSail) repository.getSail()).setParallelValidation(true);
+		((ShaclSail) repository.getSail()).setCacheSelectNodes(true);
+
+		try (SailRepositoryConnection connection = repository.getConnection()) {
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.prepareUpdate(transaction1).execute();
+			connection.commit();
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.prepareUpdate(transaction2).execute();
+			connection.commit();
+
+		}
+
+
+	}
+
+
+	@Benchmark
+	public void shaclNothingToValidateTransactionsPreloaded() {
+
+		((ShaclSail) repository.getSail()).setParallelValidation(false);
+
+		try (SailRepositoryConnection connection = repository.getConnection()) {
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.add(connection.getValueFactory().createBNode(), RDFS.LABEL,
+				connection.getValueFactory().createLiteral(""));
+			connection.commit();
+
+		}
+
+
+	}
+
+	@Benchmark
+	public void shaclParallelPreloaded() {
+
+
+		((ShaclSail) repository.getSail()).setParallelValidation(true);
+		((ShaclSail) repository.getSail()).setCacheSelectNodes(false);
+
+		try (SailRepositoryConnection connection = repository.getConnection()) {
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.prepareUpdate(transaction1).execute();
+			connection.commit();
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.prepareUpdate(transaction2).execute();
+			connection.commit();
+
+		}
+
+
+	}
+
+	@Benchmark
+	public void shaclCachePreloaded() {
+
+
+		((ShaclSail) repository.getSail()).setParallelValidation(false);
+
+		try (SailRepositoryConnection connection = repository.getConnection()) {
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.prepareUpdate(transaction1).execute();
+			connection.commit();
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.prepareUpdate(transaction2).execute();
+			connection.commit();
+
+		}
+
+
+	}
+
+	@Benchmark
+	public void shaclPreloaded() {
+
+
+		((ShaclSail) repository.getSail()).setParallelValidation(false);
+		((ShaclSail) repository.getSail()).setCacheSelectNodes(false);
+
+		try (SailRepositoryConnection connection = repository.getConnection()) {
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.prepareUpdate(transaction1).execute();
+			connection.commit();
+
+			connection.begin(IsolationLevels.SNAPSHOT);
+			connection.prepareUpdate(transaction2).execute();
+			connection.commit();
+
+		}
+
+
+	}
+
+//	@Benchmark
+//	public void noPreloading() {
+//
+//		System.out.println(".");
+//		try {
+//			SailRepository repository = new SailRepository(Utils.getInitializedShaclSail("complexBenchmark/shacl.ttl"));
+//
+//			((ShaclSail) repository.getSail()).setParallelValidation(true);
+//			((ShaclSail) repository.getSail()).setCacheSelectNodes(true);
+//
+//			try (SailRepositoryConnection connection = repository.getConnection()) {
+//				connection.begin(IsolationLevels.NONE);
+//				try (InputStream resourceAsStream = new BufferedInputStream(ComplexLargeBenchmark.class.getClassLoader().getResourceAsStream("complexBenchmark/datagovbe-valid.ttl"))) {
+//					connection.add(resourceAsStream, "", RDFFormat.TURTLE);
+//				}
+//				connection.commit();
+//			}
+//
+//		} catch (IOException e) {
+//			throw new RuntimeException(e);
+//		}
+//
+//	}
+
+}