Skip to content

Commit

Permalink
fix: continue the warmup when one configuration fails
Browse files Browse the repository at this point in the history
zepfred authored and triceo committed Jul 5, 2024
1 parent dee8cff commit 24b3240
Showing 12 changed files with 203 additions and 46 deletions.
Original file line number Diff line number Diff line change
@@ -225,7 +225,7 @@ private void warmUp(Map<Future<SubSingleBenchmarkRunner>, SubSingleBenchmarkRunn
if (firstFailureSubSingleBenchmarkRunner == null) {
firstFailureSubSingleBenchmarkRunner = subSingleBenchmarkRunner;
}
break; // Exit the warm-up loop in case of a failure.
continue; // continue to the next task
}

SolverBenchmarkResult solverBenchmarkResult = subSingleBenchmarkRunner.getSubSingleBenchmarkResult()
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package ai.timefold.solver.quarkus.benchmark.it;

import java.util.Arrays;
import java.util.List;

import jakarta.inject.Inject;
import jakarta.ws.rs.POST;
@@ -9,7 +9,10 @@
import jakarta.ws.rs.core.MediaType;

import ai.timefold.solver.benchmark.api.PlannerBenchmark;
import ai.timefold.solver.benchmark.api.PlannerBenchmarkException;
import ai.timefold.solver.benchmark.api.PlannerBenchmarkFactory;
import ai.timefold.solver.benchmark.impl.DefaultPlannerBenchmark;
import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataListValueShadowEntity;
import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowEntity;
import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowSolution;

@@ -28,11 +31,17 @@ public TimefoldBenchmarkTestResource(PlannerBenchmarkFactory benchmarkFactory) {
@Produces(MediaType.TEXT_PLAIN)
public String benchmark() {
TestdataStringLengthShadowSolution planningProblem = new TestdataStringLengthShadowSolution();
planningProblem.setEntityList(Arrays.asList(
new TestdataStringLengthShadowEntity(),
new TestdataStringLengthShadowEntity()));
planningProblem.setValueList(Arrays.asList("a", "bb", "ccc"));
planningProblem.setEntityList(List.of(
new TestdataStringLengthShadowEntity(1L),
new TestdataStringLengthShadowEntity(2L)));
planningProblem.setValueList(List.of(new TestdataListValueShadowEntity("a"), new TestdataListValueShadowEntity("bb"),
new TestdataListValueShadowEntity("ccc")));
PlannerBenchmark benchmark = benchmarkFactory.buildPlannerBenchmark(planningProblem);
return benchmark.benchmark().toPath().toAbsolutePath().toString();
try {
return benchmark.benchmark().toPath().toAbsolutePath().toString();
} catch (PlannerBenchmarkException e) {
// ignore the exception
return ((DefaultPlannerBenchmark) benchmark).getBenchmarkDirectory().getAbsolutePath();
}
}
}
Original file line number Diff line number Diff line change
@@ -4,47 +4,51 @@
import ai.timefold.solver.core.api.score.director.ScoreDirector;

public class StringLengthVariableListener
implements VariableListener<TestdataStringLengthShadowSolution, TestdataStringLengthShadowEntity> {
implements VariableListener<TestdataStringLengthShadowSolution, TestdataListValueShadowEntity> {

@Override
public void beforeEntityAdded(ScoreDirector<TestdataStringLengthShadowSolution> scoreDirector,
TestdataStringLengthShadowEntity entity) {
TestdataListValueShadowEntity entity) {
/* Nothing to do */
}

@Override
public void afterEntityAdded(ScoreDirector<TestdataStringLengthShadowSolution> scoreDirector,
TestdataStringLengthShadowEntity entity) {
TestdataListValueShadowEntity entity) {
/* Nothing to do */
}

@Override
public void beforeVariableChanged(ScoreDirector<TestdataStringLengthShadowSolution> scoreDirector,
TestdataStringLengthShadowEntity entity) {
TestdataListValueShadowEntity entity) {
/* Nothing to do */
}

@Override
public void afterVariableChanged(ScoreDirector<TestdataStringLengthShadowSolution> scoreDirector,
TestdataStringLengthShadowEntity entity) {
TestdataListValueShadowEntity entity) {
int oldLength = (entity.getLength() != null) ? entity.getLength() : 0;
int newLength = getLength(entity.getValue());
int newLength =
entity.getEntity() != null
? entity.getEntity().getValues().stream().map(TestdataListValueShadowEntity::getValue)
.mapToInt(StringLengthVariableListener::getLength).sum()
: 0;
if (oldLength != newLength) {
scoreDirector.beforeVariableChanged(entity, "length");
entity.setLength(getLength(entity.getValue()));
entity.setLength(newLength);
scoreDirector.afterVariableChanged(entity, "length");
}
}

@Override
public void beforeEntityRemoved(ScoreDirector<TestdataStringLengthShadowSolution> scoreDirector,
TestdataStringLengthShadowEntity entity) {
TestdataListValueShadowEntity entity) {
/* Nothing to do */
}

@Override
public void afterEntityRemoved(ScoreDirector<TestdataStringLengthShadowSolution> scoreDirector,
TestdataStringLengthShadowEntity entity) {
TestdataListValueShadowEntity entity) {
/* Nothing to do */
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package ai.timefold.solver.quarkus.benchmark.it.domain;

import ai.timefold.solver.core.api.domain.entity.PlanningEntity;
import ai.timefold.solver.core.api.domain.variable.InverseRelationShadowVariable;
import ai.timefold.solver.core.api.domain.variable.ShadowVariable;

@PlanningEntity
public class TestdataListValueShadowEntity {

private String value;

@InverseRelationShadowVariable(sourceVariableName = "values")
private TestdataStringLengthShadowEntity entity;

@ShadowVariable(variableListenerClass = StringLengthVariableListener.class, sourceVariableName = "entity")
private Integer length;

public TestdataListValueShadowEntity() {
}

public TestdataListValueShadowEntity(String value) {
this.value = value;
}

public String getValue() {
return value;
}

public void setValue(String value) {
this.value = value;
}

public TestdataStringLengthShadowEntity getEntity() {
return entity;
}

public void setEntity(TestdataStringLengthShadowEntity entity) {
this.entity = entity;
}

public Integer getLength() {
return length;
}

public void setLength(Integer length) {
this.length = length;
}
}
Original file line number Diff line number Diff line change
@@ -1,37 +1,43 @@
package ai.timefold.solver.quarkus.benchmark.it.domain;

import java.util.ArrayList;
import java.util.List;

import ai.timefold.solver.core.api.domain.entity.PlanningEntity;
import ai.timefold.solver.core.api.domain.variable.PlanningVariable;
import ai.timefold.solver.core.api.domain.variable.ShadowVariable;
import ai.timefold.solver.core.api.domain.lookup.PlanningId;
import ai.timefold.solver.core.api.domain.variable.PlanningListVariable;

@PlanningEntity
public class TestdataStringLengthShadowEntity {

@PlanningVariable(valueRangeProviderRefs = "valueRange")
private String value;
@PlanningId
private Long id;

@PlanningListVariable
private List<TestdataListValueShadowEntity> values;

@ShadowVariable(variableListenerClass = StringLengthVariableListener.class,
sourceEntityClass = TestdataStringLengthShadowEntity.class, sourceVariableName = "value")
private Integer length;
public TestdataStringLengthShadowEntity() {
}

public TestdataStringLengthShadowEntity(Long id) {
this.id = id;
this.values = new ArrayList<>();
}

// ************************************************************************
// Getters/setters
// ************************************************************************

public String getValue() {
return value;
}

public void setValue(String value) {
this.value = value;
public Long getId() {
return id;
}

public Integer getLength() {
return length;
public List<TestdataListValueShadowEntity> getValues() {
return values;
}

public void setLength(Integer length) {
this.length = length;
public void setValues(List<TestdataListValueShadowEntity> values) {
this.values = values;
}

}
Original file line number Diff line number Diff line change
@@ -5,14 +5,18 @@
import ai.timefold.solver.core.api.domain.solution.PlanningEntityCollectionProperty;
import ai.timefold.solver.core.api.domain.solution.PlanningScore;
import ai.timefold.solver.core.api.domain.solution.PlanningSolution;
import ai.timefold.solver.core.api.domain.solution.ProblemFactCollectionProperty;
import ai.timefold.solver.core.api.domain.valuerange.ValueRangeProvider;
import ai.timefold.solver.core.api.score.buildin.hardsoft.HardSoftScore;

@PlanningSolution
public class TestdataStringLengthShadowSolution {

@ValueRangeProvider(id = "valueRange")
private List<String> valueList;

@ProblemFactCollectionProperty
@ValueRangeProvider
private List<TestdataListValueShadowEntity> valueList;

@PlanningEntityCollectionProperty
private List<TestdataStringLengthShadowEntity> entityList;

@@ -23,11 +27,11 @@ public class TestdataStringLengthShadowSolution {
// Getters/setters
// ************************************************************************

public List<String> getValueList() {
public List<TestdataListValueShadowEntity> getValueList() {
return valueList;
}

public void setValueList(List<String> valueList) {
public void setValueList(List<TestdataListValueShadowEntity> valueList) {
this.valueList = valueList;
}

Original file line number Diff line number Diff line change
@@ -4,21 +4,20 @@
import ai.timefold.solver.core.api.score.stream.Constraint;
import ai.timefold.solver.core.api.score.stream.ConstraintFactory;
import ai.timefold.solver.core.api.score.stream.ConstraintProvider;
import ai.timefold.solver.core.api.score.stream.Joiners;
import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataListValueShadowEntity;
import ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowEntity;

public class TestdataStringLengthConstraintProvider implements ConstraintProvider {

@Override
public Constraint[] defineConstraints(ConstraintFactory factory) {
return new Constraint[] {
factory.forEach(TestdataStringLengthShadowEntity.class)
.join(TestdataStringLengthShadowEntity.class, Joiners.equal(TestdataStringLengthShadowEntity::getValue))
.filter((a, b) -> a != b)
factory.forEachUniquePair(TestdataStringLengthShadowEntity.class)
.filter((a, b) -> a.getValues().stream().anyMatch(v -> b.getValues().contains(v)))
.penalize(HardSoftScore.ONE_HARD)
.asConstraint("Don't assign 2 entities the same value."),
factory.forEach(TestdataStringLengthShadowEntity.class)
.reward(HardSoftScore.ONE_SOFT, TestdataStringLengthShadowEntity::getLength)
factory.forEach(TestdataListValueShadowEntity.class)
.reward(HardSoftScore.ONE_SOFT, a -> a.getLength())
.asConstraint("Maximize value length")
};
}
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# The solver runs to a known best score to avoid a HTTP timeout in this simple implementation.
quarkus.timefold.benchmark.solver.termination.best-score-limit=0hard/5soft
quarkus.timefold.benchmark.solver.termination.best-score-limit=0hard/10soft
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package ai.timefold.solver.quarkus.benchmark.it;

import static org.apache.http.params.CoreConnectionPNames.SO_TIMEOUT;
import static org.assertj.core.api.Assertions.assertThat;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;

import org.junit.jupiter.api.Test;

import io.quarkus.test.junit.QuarkusTest;
import io.quarkus.test.junit.QuarkusTestProfile;
import io.quarkus.test.junit.TestProfile;
import io.restassured.RestAssured;
import io.restassured.config.HttpClientConfig;
import io.restassured.config.RestAssuredConfig;
import io.restassured.path.xml.XmlPath;

/**
* Test the benchmark for typical solver configs (blueprint).
*/
@QuarkusTest
@TestProfile(TimefoldBenchmarkBlueprintTest.BlueprintTestProfile.class)
class TimefoldBenchmarkBlueprintTest {

@Test
void benchmark() throws Exception {
RestAssuredConfig timeoutConfig = RestAssured.config()
.httpClient(HttpClientConfig.httpClientConfig()
.setParam(SO_TIMEOUT, 10000));
String benchmarkResultDirectory = RestAssured
.given()
.config(timeoutConfig)
.header("Content-Type", "text/plain;charset=UTF-8")
.when()
.post("/timefold/test/benchmark")
.body().asString();
assertThat(benchmarkResultDirectory).isNotNull();
Path benchmarkResultDirectoryPath = Path.of(benchmarkResultDirectory);
assertThat(Files.isDirectory(benchmarkResultDirectoryPath)).isTrue();
Path benchmarkResultPath = Files.walk(benchmarkResultDirectoryPath, 2)
.filter(path -> path.endsWith("plannerBenchmarkResult.xml")).findFirst().orElseThrow();
assertThat(Files.isRegularFile(benchmarkResultPath)).isTrue();
XmlPath xmlPath = XmlPath.from(benchmarkResultPath.toFile());
assertThat(xmlPath
.getList(
"plannerBenchmarkResult.solverBenchmarkResult.singleBenchmarkResult.subSingleBenchmarkResult.succeeded")
.stream().anyMatch(node -> node.equals("true")))
.isTrue();
}

public static class BlueprintTestProfile implements QuarkusTestProfile {

@Override
public Map<String, String> getConfigOverrides() {
Map<String, String> overrides = new HashMap<>(QuarkusTestProfile.super.getConfigOverrides());
overrides.put("quarkus.timefold.benchmark.solver-benchmark-config-xml", "blueprintSolverBenchmarkConfig.xml");
return overrides;
}
}
}
Original file line number Diff line number Diff line change
@@ -9,6 +9,6 @@
*/
@QuarkusIntegrationTest
@Disabled("timefold-solver-quarkus-benchmark cannot compile to native")
public class TimefoldBenchmarkTestResourceIT extends TimefoldBenchmarkTestResourceTest {
public class TimefoldBenchmarkResourceIT extends TimefoldBenchmarkResourceTest {

}
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@
*/

@QuarkusTest
class TimefoldBenchmarkTestResourceTest {
class TimefoldBenchmarkResourceTest {

@Test
@Timeout(600)
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<plannerBenchmark xmlns="https://timefold.ai/xsd/benchmark"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://timefold.ai/xsd/benchmark https://timefold.ai/xsd/benchmark/benchmark.xsd">
<parallelBenchmarkCount>AUTO</parallelBenchmarkCount>

<inheritedSolverBenchmark>
<solver>
<solutionClass>ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowSolution</solutionClass>
<entityClass>ai.timefold.solver.quarkus.benchmark.it.domain.TestdataStringLengthShadowEntity</entityClass>
<entityClass>ai.timefold.solver.quarkus.benchmark.it.domain.TestdataListValueShadowEntity</entityClass>
<scoreDirectorFactory>
<constraintProviderClass>ai.timefold.solver.quarkus.benchmark.it.solver.TestdataStringLengthConstraintProvider</constraintProviderClass>
</scoreDirectorFactory>
<termination>
<secondsSpentLimit>5</secondsSpentLimit>
</termination>
</solver>
</inheritedSolverBenchmark>

<solverBenchmarkBluePrint>
<solverBenchmarkBluePrintType>EVERY_CONSTRUCTION_HEURISTIC_TYPE_WITH_EVERY_LOCAL_SEARCH_TYPE</solverBenchmarkBluePrintType>
</solverBenchmarkBluePrint>
</plannerBenchmark>

0 comments on commit 24b3240

Please sign in to comment.