From b14201cfc8e7eabf62158d849738f1fd5ba89e14 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Tue, 16 Jul 2024 10:42:49 +0200 Subject: [PATCH] Added Memory Limitation Exploration --- .gitignore | 1 + build.sbt | 2 -- runBenchmarks.sc | 36 ++++++++++++------- .../com/github/plume/oss/Benchmark.scala | 4 +-- .../scala/com/github/plume/oss/Plume.scala | 9 +++++ .../TinkerGraphReadBenchmark.scala | 7 ++-- .../scala/com/github/plume/oss/package.scala | 1 + 7 files changed, 39 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 65ffdbe6..0bdf399b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ graph.xml gsql_client.* *.txt *.csv +.local /workspace /results diff --git a/build.sbt b/build.sbt index ac462aba..4fc79376 100644 --- a/build.sbt +++ b/build.sbt @@ -69,8 +69,6 @@ publishMavenStyle := true lazy val datasetDir = taskKey[File]("Dataset directory") datasetDir := baseDirectory.value / "workspace" / "defects4j" -lazy val driversToBenchmark = taskKey[Seq[String]]("Drivers to benchmark") -driversToBenchmark := Seq("overflowdb", "tinkergraph", "neo4j-embedded") lazy val defect4jDataset = taskKey[Seq[(String, String)]]("JARs for projects used in `defects4j`") defect4jDataset := diff --git a/runBenchmarks.sc b/runBenchmarks.sc index 49607ce8..1f6e2988 100644 --- a/runBenchmarks.sc +++ b/runBenchmarks.sc @@ -8,16 +8,15 @@ import scala.jdk.CollectionConverters.* "sbt compile benchmarkDownloadTask".! val datasetDir = Path.of("workspace", "defects4j") - val resultsDir = Path.of("results") + val resultsDir = Path.of("results").createIfNotExists - if (!Files.exists(resultsDir)) Files.createDirectory(resultsDir) - - def benchmarkArgs(driver: String, project: String): String = { + def benchmarkArgs(driver: String, project: String, memGb: Int): String = { val projectDir = Path.of(datasetDir.toString, project) val projectName = project.toLowerCase.stripSuffix(".jar") - val resultsPath = Path.of(resultsDir.toString, s"results-$driver-$projectName") - val outputPath = Path.of(resultsDir.toString, s"output-$driver-$projectName") - s"Jmh/runMain com.github.plume.oss.Benchmark $driver $projectDir -o ${outputPath.toAbsolutePath} -r ${resultsPath.toAbsolutePath}" + val driverResultsDir = Path.of(resultsDir.toString, driver, projectName).createIfNotExists + val resultsPath = Path.of(driverResultsDir.toString, s"results-Xmx${memGb}G") + val outputPath = Path.of(driverResultsDir.toString, s"output-Xmx${memGb}G") + s"Jmh/runMain com.github.plume.oss.Benchmark $driver $projectDir -o ${outputPath.toAbsolutePath} -r ${resultsPath.toAbsolutePath} -m $memGb" } println("[info] Available projects:") @@ -25,14 +24,25 @@ import scala.jdk.CollectionConverters.* projects.foreach(p => println(s" - ${p.getFileName.toString}")) println("[info] Available drivers:") - val drivers = Seq("overflowdb") + val drivers = Seq("overflowdb", "tinkergraph", "neo4j-embedded") drivers.foreach(d => println(s" - $d")) - drivers.foreach { driver => - projects.foreach { project => - val cmd = benchmarkArgs(driver, project.getFileName.toString) - println(s"[info] Benchmarking '$driver' on project '$project'") - s"sbt \"$cmd\"".! + val memoryConfigs = Seq(4, 8, 16) + + memoryConfigs.foreach { memConfig => + drivers.foreach { driver => + projects.foreach { project => + val cmd = benchmarkArgs(driver, project.getFileName.toString, memConfig) + println(s"[info] Benchmarking '$driver' on project '$project' with `-Xmx${memConfig}G`") + s"sbt \"$cmd\"".! + } } } } + +implicit class PathExt(x: Path) { + def createIfNotExists: Path = { + if (!Files.exists(x)) Files.createDirectories(x) + x + } +} \ No newline at end of file diff --git a/src/main/scala/com/github/plume/oss/Benchmark.scala b/src/main/scala/com/github/plume/oss/Benchmark.scala index 8538bfdb..3b7213d7 100644 --- a/src/main/scala/com/github/plume/oss/Benchmark.scala +++ b/src/main/scala/com/github/plume/oss/Benchmark.scala @@ -72,14 +72,14 @@ object Benchmark { .warmupIterations(1) .warmupTime(TimeValue.seconds(1)) .measurementTime(TimeValue.seconds(2)) - .measurementIterations(3) + .measurementIterations(5) .mode(Mode.AverageTime) .timeUnit(TimeUnit.NANOSECONDS) .forks(2) .output(s"${config.jmhOutputFile}-$benchmarkType.txt") .result(s"${config.jmhResultFile}-$benchmarkType.csv") .param("configStr", write(config)) - .detectJvmArgs() // inherit stuff like max heap size + .jvmArgsAppend(s"-Xmx${config.jmhMemoryGb}G", "-XX:+UseZGC") } enum BenchmarkType { diff --git a/src/main/scala/com/github/plume/oss/Plume.scala b/src/main/scala/com/github/plume/oss/Plume.scala index 66324344..ce155dfc 100644 --- a/src/main/scala/com/github/plume/oss/Plume.scala +++ b/src/main/scala/com/github/plume/oss/Plume.scala @@ -44,6 +44,15 @@ object Plume { .text("The target application to parse.") .action((x, c) => c.copy(inputDir = x)) + opt[Int]('m', "jmh-memory") + .text(s"The -Xmx JVM argument in Gb for JMH. Default is 4 (-Xmx4G).") + .hidden() + .validate { + case x if x < 1 => failure("Consider at least 1Gb") + case _ => success + } + .action((x, c) => c.copy(jmhMemoryGb = x)) + opt[String]('o', "jmh-output-file") .text(s"The JMH output file path. Exclude file extensions.") .hidden() diff --git a/src/main/scala/com/github/plume/oss/benchmarking/TinkerGraphReadBenchmark.scala b/src/main/scala/com/github/plume/oss/benchmarking/TinkerGraphReadBenchmark.scala index 999c2516..9b084973 100644 --- a/src/main/scala/com/github/plume/oss/benchmarking/TinkerGraphReadBenchmark.scala +++ b/src/main/scala/com/github/plume/oss/benchmarking/TinkerGraphReadBenchmark.scala @@ -1,19 +1,18 @@ package com.github.plume.oss.benchmarking -import com.github.plume.oss.benchmarking.GraphReadBenchmark import com.github.plume.oss.drivers.TinkerGraphDriver import io.shiftleft.codepropertygraph.generated.EdgeTypes.AST import io.shiftleft.codepropertygraph.generated.NodeTypes.{CALL, METHOD} import io.shiftleft.codepropertygraph.generated.PropertyNames.{FULL_NAME, ORDER} import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.{GraphTraversalSource, __} -import org.openjdk.jmh.annotations.{Benchmark, Scope, Setup, State} +import org.openjdk.jmh.annotations.* import org.openjdk.jmh.infra.{BenchmarkParams, Blackhole} +import overflowdb.traversal.* -import scala.jdk.CollectionConverters.* import scala.compiletime.uninitialized +import scala.jdk.CollectionConverters.* import scala.util.Random -import overflowdb.traversal.* @State(Scope.Benchmark) class TinkerGraphReadBenchmark extends GraphReadBenchmark { diff --git a/src/main/scala/com/github/plume/oss/package.scala b/src/main/scala/com/github/plume/oss/package.scala index 689cbb4a..7dceda43 100644 --- a/src/main/scala/com/github/plume/oss/package.scala +++ b/src/main/scala/com/github/plume/oss/package.scala @@ -8,6 +8,7 @@ package object oss { case class PlumeConfig( inputDir: String = "", + jmhMemoryGb: Int = 4, jmhOutputFile: String = File.newTemporaryFile("plume-jmh-output-").pathAsString, jmhResultFile: String = File.newTemporaryFile("plume-jmh-result-").pathAsString, dbConfig: DatabaseConfig = OverflowDbConfig()