From 24b32407cbc65cd59cd607f65eb457996571ae86 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 5 Jul 2024 09:01:00 -0300 Subject: [PATCH] fix: continue the warmup when one configuration fails --- .../impl/DefaultPlannerBenchmark.java | 2 +- .../it/TimefoldBenchmarkTestResource.java | 21 +++++-- .../domain/StringLengthVariableListener.java | 22 ++++--- .../domain/TestdataListValueShadowEntity.java | 48 ++++++++++++++ .../TestdataStringLengthShadowEntity.java | 40 +++++++----- .../TestdataStringLengthShadowSolution.java | 12 ++-- ...estdataStringLengthConstraintProvider.java | 11 ++-- .../src/main/resources/application.properties | 2 +- .../it/TimefoldBenchmarkBlueprintTest.java | 63 +++++++++++++++++++ ....java => TimefoldBenchmarkResourceIT.java} | 2 +- ...ava => TimefoldBenchmarkResourceTest.java} | 2 +- .../blueprintSolverBenchmarkConfig.xml | 24 +++++++ 12 files changed, 203 insertions(+), 46 deletions(-) create mode 100644 quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataListValueShadowEntity.java create mode 100644 quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkBlueprintTest.java rename quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/{TimefoldBenchmarkTestResourceIT.java => TimefoldBenchmarkResourceIT.java} (77%) rename quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/{TimefoldBenchmarkTestResourceTest.java => TimefoldBenchmarkResourceTest.java} (97%) create mode 100644 quarkus-integration/quarkus-benchmark/integration-test/src/test/resources/blueprintSolverBenchmarkConfig.xml diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/DefaultPlannerBenchmark.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/DefaultPlannerBenchmark.java index 26ef6e399a..4c4212d32f 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/DefaultPlannerBenchmark.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/DefaultPlannerBenchmark.java @@ -225,7 +225,7 @@ private void warmUp(Map, SubSingleBenchmarkRunn if (firstFailureSubSingleBenchmarkRunner == null) { firstFailureSubSingleBenchmarkRunner = subSingleBenchmarkRunner; } - break; // Exit the warm-up loop in case of a failure. + continue; // continue to the next task } SolverBenchmarkResult solverBenchmarkResult = subSingleBenchmarkRunner.getSubSingleBenchmarkResult() diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResource.java b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResource.java index b7a9f31086..e4aa45518c 100644 --- a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResource.java +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResource.java @@ -1,6 +1,6 @@ package ai.timefold.solver.quarkus.benchmark.it; -import java.util.Arrays; +import java.util.List; import jakarta.inject.Inject; import jakarta.ws.rs.POST; @@ -9,7 +9,10 @@ import jakarta.ws.rs.core.MediaType; import ai.timefold.solver.benchmark.api.PlannerBenchmark; +import ai.timefold.solver.benchmark.api.PlannerBenchmarkException; import ai.timefold.solver.benchmark.api.PlannerBenchmarkFactory; +import ai.timefold.solver.benchmark.impl.DefaultPlannerBenchmark; +import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataListValueShadowEntity; import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowEntity; import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowSolution; @@ -28,11 +31,17 @@ public TimefoldBenchmarkTestResource(PlannerBenchmarkFactory benchmarkFactory) { @Produces(MediaType.TEXT_PLAIN) public String benchmark() { TestdataStringLengthShadowSolution planningProblem = new TestdataStringLengthShadowSolution(); - planningProblem.setEntityList(Arrays.asList( - new TestdataStringLengthShadowEntity(), - new TestdataStringLengthShadowEntity())); - planningProblem.setValueList(Arrays.asList("a", "bb", "ccc")); + planningProblem.setEntityList(List.of( + new TestdataStringLengthShadowEntity(1L), + new TestdataStringLengthShadowEntity(2L))); + planningProblem.setValueList(List.of(new TestdataListValueShadowEntity("a"), new TestdataListValueShadowEntity("bb"), + new TestdataListValueShadowEntity("ccc"))); PlannerBenchmark benchmark = benchmarkFactory.buildPlannerBenchmark(planningProblem); - return benchmark.benchmark().toPath().toAbsolutePath().toString(); + try { + return benchmark.benchmark().toPath().toAbsolutePath().toString(); + } catch (PlannerBenchmarkException e) { + // ignore the exception + return ((DefaultPlannerBenchmark) benchmark).getBenchmarkDirectory().getAbsolutePath(); + } } } diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/StringLengthVariableListener.java b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/StringLengthVariableListener.java index e179116330..2753a77d62 100644 --- a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/StringLengthVariableListener.java +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/StringLengthVariableListener.java @@ -4,47 +4,51 @@ import ai.timefold.solver.core.api.score.director.ScoreDirector; public class StringLengthVariableListener - implements VariableListener { + implements VariableListener { @Override public void beforeEntityAdded(ScoreDirector scoreDirector, - TestdataStringLengthShadowEntity entity) { + TestdataListValueShadowEntity entity) { /* Nothing to do */ } @Override public void afterEntityAdded(ScoreDirector scoreDirector, - TestdataStringLengthShadowEntity entity) { + TestdataListValueShadowEntity entity) { /* Nothing to do */ } @Override public void beforeVariableChanged(ScoreDirector scoreDirector, - TestdataStringLengthShadowEntity entity) { + TestdataListValueShadowEntity entity) { /* Nothing to do */ } @Override public void afterVariableChanged(ScoreDirector scoreDirector, - TestdataStringLengthShadowEntity entity) { + TestdataListValueShadowEntity entity) { int oldLength = (entity.getLength() != null) ? entity.getLength() : 0; - int newLength = getLength(entity.getValue()); + int newLength = + entity.getEntity() != null + ? entity.getEntity().getValues().stream().map(TestdataListValueShadowEntity::getValue) + .mapToInt(StringLengthVariableListener::getLength).sum() + : 0; if (oldLength != newLength) { scoreDirector.beforeVariableChanged(entity, "length"); - entity.setLength(getLength(entity.getValue())); + entity.setLength(newLength); scoreDirector.afterVariableChanged(entity, "length"); } } @Override public void beforeEntityRemoved(ScoreDirector scoreDirector, - TestdataStringLengthShadowEntity entity) { + TestdataListValueShadowEntity entity) { /* Nothing to do */ } @Override public void afterEntityRemoved(ScoreDirector scoreDirector, - TestdataStringLengthShadowEntity entity) { + TestdataListValueShadowEntity entity) { /* Nothing to do */ } diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataListValueShadowEntity.java b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataListValueShadowEntity.java new file mode 100644 index 0000000000..41f8269399 --- /dev/null +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataListValueShadowEntity.java @@ -0,0 +1,48 @@ +package ai.timefold.solver.quarkus.benchmark.it.domain; + +import ai.timefold.solver.core.api.domain.entity.PlanningEntity; +import ai.timefold.solver.core.api.domain.variable.InverseRelationShadowVariable; +import ai.timefold.solver.core.api.domain.variable.ShadowVariable; + +@PlanningEntity +public class TestdataListValueShadowEntity { + + private String value; + + @InverseRelationShadowVariable(sourceVariableName = "values") + private TestdataStringLengthShadowEntity entity; + + @ShadowVariable(variableListenerClass = StringLengthVariableListener.class, sourceVariableName = "entity") + private Integer length; + + public TestdataListValueShadowEntity() { + } + + public TestdataListValueShadowEntity(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public TestdataStringLengthShadowEntity getEntity() { + return entity; + } + + public void setEntity(TestdataStringLengthShadowEntity entity) { + this.entity = entity; + } + + public Integer getLength() { + return length; + } + + public void setLength(Integer length) { + this.length = length; + } +} diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataStringLengthShadowEntity.java b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataStringLengthShadowEntity.java index 85532edad0..fcefede48c 100644 --- a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataStringLengthShadowEntity.java +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataStringLengthShadowEntity.java @@ -1,37 +1,43 @@ package ai.timefold.solver.quarkus.benchmark.it.domain; +import java.util.ArrayList; +import java.util.List; + import ai.timefold.solver.core.api.domain.entity.PlanningEntity; -import ai.timefold.solver.core.api.domain.variable.PlanningVariable; -import ai.timefold.solver.core.api.domain.variable.ShadowVariable; +import ai.timefold.solver.core.api.domain.lookup.PlanningId; +import ai.timefold.solver.core.api.domain.variable.PlanningListVariable; @PlanningEntity public class TestdataStringLengthShadowEntity { - @PlanningVariable(valueRangeProviderRefs = "valueRange") - private String value; + @PlanningId + private Long id; + + @PlanningListVariable + private List values; - @ShadowVariable(variableListenerClass = StringLengthVariableListener.class, - sourceEntityClass = TestdataStringLengthShadowEntity.class, sourceVariableName = "value") - private Integer length; + public TestdataStringLengthShadowEntity() { + } + + public TestdataStringLengthShadowEntity(Long id) { + this.id = id; + this.values = new ArrayList<>(); + } // ************************************************************************ // Getters/setters // ************************************************************************ - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; + public Long getId() { + return id; } - public Integer getLength() { - return length; + public List getValues() { + return values; } - public void setLength(Integer length) { - this.length = length; + public void setValues(List values) { + this.values = values; } } diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataStringLengthShadowSolution.java b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataStringLengthShadowSolution.java index 358cf9554e..0bbfaa0ab0 100644 --- a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataStringLengthShadowSolution.java +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/domain/TestdataStringLengthShadowSolution.java @@ -5,14 +5,18 @@ import ai.timefold.solver.core.api.domain.solution.PlanningEntityCollectionProperty; import ai.timefold.solver.core.api.domain.solution.PlanningScore; import ai.timefold.solver.core.api.domain.solution.PlanningSolution; +import ai.timefold.solver.core.api.domain.solution.ProblemFactCollectionProperty; import ai.timefold.solver.core.api.domain.valuerange.ValueRangeProvider; import ai.timefold.solver.core.api.score.buildin.hardsoft.HardSoftScore; @PlanningSolution public class TestdataStringLengthShadowSolution { - @ValueRangeProvider(id = "valueRange") - private List valueList; + + @ProblemFactCollectionProperty + @ValueRangeProvider + private List valueList; + @PlanningEntityCollectionProperty private List entityList; @@ -23,11 +27,11 @@ public class TestdataStringLengthShadowSolution { // Getters/setters // ************************************************************************ - public List getValueList() { + public List getValueList() { return valueList; } - public void setValueList(List valueList) { + public void setValueList(List valueList) { this.valueList = valueList; } diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/solver/TestdataStringLengthConstraintProvider.java b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/solver/TestdataStringLengthConstraintProvider.java index 3adf414254..6a06886522 100644 --- a/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/solver/TestdataStringLengthConstraintProvider.java +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/main/java/ai/timefold/solver/quarkus/benchmark/it/solver/TestdataStringLengthConstraintProvider.java @@ -4,7 +4,7 @@ import ai.timefold.solver.core.api.score.stream.Constraint; import ai.timefold.solver.core.api.score.stream.ConstraintFactory; import ai.timefold.solver.core.api.score.stream.ConstraintProvider; -import ai.timefold.solver.core.api.score.stream.Joiners; +import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataListValueShadowEntity; import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowEntity; public class TestdataStringLengthConstraintProvider implements ConstraintProvider { @@ -12,13 +12,12 @@ public class TestdataStringLengthConstraintProvider implements ConstraintProvide @Override public Constraint[] defineConstraints(ConstraintFactory factory) { return new Constraint[] { - factory.forEach(TestdataStringLengthShadowEntity.class) - .join(TestdataStringLengthShadowEntity.class, Joiners.equal(TestdataStringLengthShadowEntity::getValue)) - .filter((a, b) -> a != b) + factory.forEachUniquePair(TestdataStringLengthShadowEntity.class) + .filter((a, b) -> a.getValues().stream().anyMatch(v -> b.getValues().contains(v))) .penalize(HardSoftScore.ONE_HARD) .asConstraint("Don't assign 2 entities the same value."), - factory.forEach(TestdataStringLengthShadowEntity.class) - .reward(HardSoftScore.ONE_SOFT, TestdataStringLengthShadowEntity::getLength) + factory.forEach(TestdataListValueShadowEntity.class) + .reward(HardSoftScore.ONE_SOFT, a -> a.getLength()) .asConstraint("Maximize value length") }; } diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/main/resources/application.properties b/quarkus-integration/quarkus-benchmark/integration-test/src/main/resources/application.properties index 25f181244d..eeac6f867f 100644 --- a/quarkus-integration/quarkus-benchmark/integration-test/src/main/resources/application.properties +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/main/resources/application.properties @@ -1,2 +1,2 @@ # The solver runs to a known best score to avoid a HTTP timeout in this simple implementation. -quarkus.timefold.benchmark.solver.termination.best-score-limit=0hard/5soft \ No newline at end of file +quarkus.timefold.benchmark.solver.termination.best-score-limit=0hard/10soft \ No newline at end of file diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkBlueprintTest.java b/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkBlueprintTest.java new file mode 100644 index 0000000000..51d500c0bc --- /dev/null +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkBlueprintTest.java @@ -0,0 +1,63 @@ +package ai.timefold.solver.quarkus.benchmark.it; + +import static org.apache.http.params.CoreConnectionPNames.SO_TIMEOUT; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import io.restassured.RestAssured; +import io.restassured.config.HttpClientConfig; +import io.restassured.config.RestAssuredConfig; +import io.restassured.path.xml.XmlPath; + +/** + * Test the benchmark for typical solver configs (blueprint). + */ +@QuarkusTest +@TestProfile(TimefoldBenchmarkBlueprintTest.BlueprintTestProfile.class) +class TimefoldBenchmarkBlueprintTest { + + @Test + void benchmark() throws Exception { + RestAssuredConfig timeoutConfig = RestAssured.config() + .httpClient(HttpClientConfig.httpClientConfig() + .setParam(SO_TIMEOUT, 10000)); + String benchmarkResultDirectory = RestAssured + .given() + .config(timeoutConfig) + .header("Content-Type", "text/plain;charset=UTF-8") + .when() + .post("/timefold/test/benchmark") + .body().asString(); + assertThat(benchmarkResultDirectory).isNotNull(); + Path benchmarkResultDirectoryPath = Path.of(benchmarkResultDirectory); + assertThat(Files.isDirectory(benchmarkResultDirectoryPath)).isTrue(); + Path benchmarkResultPath = Files.walk(benchmarkResultDirectoryPath, 2) + .filter(path -> path.endsWith("plannerBenchmarkResult.xml")).findFirst().orElseThrow(); + assertThat(Files.isRegularFile(benchmarkResultPath)).isTrue(); + XmlPath xmlPath = XmlPath.from(benchmarkResultPath.toFile()); + assertThat(xmlPath + .getList( + "plannerBenchmarkResult.solverBenchmarkResult.singleBenchmarkResult.subSingleBenchmarkResult.succeeded") + .stream().anyMatch(node -> node.equals("true"))) + .isTrue(); + } + + public static class BlueprintTestProfile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + Map overrides = new HashMap<>(QuarkusTestProfile.super.getConfigOverrides()); + overrides.put("quarkus.timefold.benchmark.solver-benchmark-config-xml", "blueprintSolverBenchmarkConfig.xml"); + return overrides; + } + } +} diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResourceIT.java b/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkResourceIT.java similarity index 77% rename from quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResourceIT.java rename to quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkResourceIT.java index da3ce122e5..08f48dce27 100644 --- a/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResourceIT.java +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkResourceIT.java @@ -9,6 +9,6 @@ */ @QuarkusIntegrationTest @Disabled("timefold-solver-quarkus-benchmark cannot compile to native") -public class TimefoldBenchmarkTestResourceIT extends TimefoldBenchmarkTestResourceTest { +public class TimefoldBenchmarkResourceIT extends TimefoldBenchmarkResourceTest { } diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResourceTest.java b/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkResourceTest.java similarity index 97% rename from quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResourceTest.java rename to quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkResourceTest.java index df3f2c052c..bd78c731f5 100644 --- a/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkTestResourceTest.java +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/test/java/ai/timefold/solver/quarkus/benchmark/it/TimefoldBenchmarkResourceTest.java @@ -17,7 +17,7 @@ */ @QuarkusTest -class TimefoldBenchmarkTestResourceTest { +class TimefoldBenchmarkResourceTest { @Test @Timeout(600) diff --git a/quarkus-integration/quarkus-benchmark/integration-test/src/test/resources/blueprintSolverBenchmarkConfig.xml b/quarkus-integration/quarkus-benchmark/integration-test/src/test/resources/blueprintSolverBenchmarkConfig.xml new file mode 100644 index 0000000000..92d2adc3be --- /dev/null +++ b/quarkus-integration/quarkus-benchmark/integration-test/src/test/resources/blueprintSolverBenchmarkConfig.xml @@ -0,0 +1,24 @@ + + + AUTO + + + + ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowSolution + ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowEntity + ai.timefold.solver.quarkus.benchmark.it.domain.TestdataListValueShadowEntity + + ai.timefold.solver.quarkus.benchmark.it.solver.TestdataStringLengthConstraintProvider + + + 5 + + + + + + EVERY_CONSTRUCTION_HEURISTIC_TYPE_WITH_EVERY_LOCAL_SEARCH_TYPE + + \ No newline at end of file