Skip to content

Commit

Permalink
feat: Improve constraint verifier error messages (#758)
Browse files Browse the repository at this point in the history 
This PR improves the error messages of the constraint verifier and adds
two new methods: `justifiesWithExactly()` and `indictsWithExactly()`.
The new messages will be formatted as follows:

## Justifications

```
Broken expectation.
     Justification: ai.timefold.solver.test.api.score.stream.testdata/Justify with first justification
               Expected:
                          TestFirstJustification[id=2]
                 Actual:
                          TestFirstJustification[id=Generated Entity 0]
                          TestFirstJustification[id=Generated Entity 1]
                          TestFirstJustification[id=Generated Entity 2]
 Expected but not found:
                          TestFirstJustification[id=2]
   Unexpected but found:
                          TestFirstJustification[id=Generated Entity 0]
                          TestFirstJustification[id=Generated Entity 1]
                          TestFirstJustification[id=Generated Entity 2]
```

## Indictments

```
Broken expectation.
        Indictment: ai.timefold.solver.test.api.score.stream.testdata/Justify with first justification
               Expected:
                          TestdataConstraintVerifierFirstEntity(code='bad code')
                 Actual:
                          TestdataConstraintVerifierFirstEntity(code='Generated Entity 0')
                          TestdataConstraintVerifierFirstEntity(code='Generated Entity 1')
                          TestdataConstraintVerifierFirstEntity(code='Generated Entity 2')
 Expected but not found:
                          TestdataConstraintVerifierFirstEntity(code='bad code')
   Unexpected but found:
                          TestdataConstraintVerifierFirstEntity(code='Generated Entity 0')
                          TestdataConstraintVerifierFirstEntity(code='Generated Entity 1')
                          TestdataConstraintVerifierFirstEntity(code='Generated Entity 2')
```
  • Loading branch information
@zepfred
zepfred authored Mar 27, 2024
1 parent 4a62ced commit a99e600
Show file tree
Hide file tree
Showing 4 changed files with 434 additions and 116 deletions.
4 changes: 2 additions & 2 deletions docs/src/modules/ROOT/pages/constraints-and-score/score-calculation.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1281,7 +1281,7 @@ Alternatively, you can use a `givenSolution(...)` method here and provide a plan

[NOTE]
====
You can also use methods `justifiesWith(...)` and `indictsWith(...)`
You can also use methods `justifiesWith(...)`, `justifiesWithExactly(...)`, `indictsWith(...)` and `indictsWithExactly(...)`
to validate the expected constraint justifications and indictments mapped by the constraint definition.
====

Expand All @@ -1300,7 +1300,7 @@ It will neither set nor update shadow variables.
If the tested constraints depend on shadow variables,
it is your responsibility to assign the correct values beforehand.
When executing `justifiesWith(...)` and `indictsWith(...)`,
When executing `justifiesWith(...)`, `justifiesWithExactly(...)`, `indictsWith(...)` and `indictsWithExactly(...)`,
comparisons are made using the standard method `equals` on the fact problem instances.
====

Expand Down
50 changes: 46 additions & 4 deletions test/src/main/java/ai/timefold/solver/test/api/score/stream/SingleConstraintAssertion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,8 @@ public interface SingleConstraintAssertion {
SingleConstraintAssertion justifiesWith(String message, ConstraintJustification... justifications);

/**
* Asserts that the {@link Constraint} being tested, given a set of facts, results in a specific
* Asserts that the {@link Constraint} being tested, given a set of facts, results in a given
* {@link ConstraintJustification}.
* <p>
* The justification class types must match; otherwise it fails with no match.
*
* @param justifications the expected justifications.
* @return never null
Expand All @@ -33,7 +31,29 @@ default SingleConstraintAssertion justifiesWith(ConstraintJustification... justi
}

/**
* Asserts that the {@link Constraint} being tested, given a set of facts, results in a specific indictments.
* As defined by {@link #justifiesWithExactly(ConstraintJustification...)}.
*
* @param justifications the expected justification.
* @param message sometimes null, description of the scenario being asserted
* @return never null
* @throws AssertionError when the expected penalty is not observed
*/
SingleConstraintAssertion justifiesWithExactly(String message, ConstraintJustification... justifications);

/**
* Asserts that the {@link Constraint} being tested, given a set of facts, results in a given
* {@link ConstraintJustification} and nothing else.
*
* @param justifications the expected justifications.
* @return never null
* @throws AssertionError when the expected penalty is not observed
*/
default SingleConstraintAssertion justifiesWithExactly(ConstraintJustification... justifications) {
return justifiesWithExactly(null, justifications);
}

/**
* Asserts that the {@link Constraint} being tested, given a set of facts, results in the given indictments.
*
* @param indictments the expected indictments.
* @return never null
Expand All @@ -53,6 +73,28 @@ default SingleConstraintAssertion indictsWith(Object... indictments) {
*/
SingleConstraintAssertion indictsWith(String message, Object... indictments);

/**
* Asserts that the {@link Constraint} being tested, given a set of facts, results in the given indictments and
* nothing else.
*
* @param indictments the expected indictments.
* @return never null
* @throws AssertionError when the expected penalty is not observed
*/
default SingleConstraintAssertion indictsWithExactly(Object... indictments) {
return indictsWithExactly(null, indictments);
}

/**
* As defined by {@link #indictsWithExactly(Object...)}.
*
* @param message sometimes null, description of the scenario being asserted
* @param indictments the expected indictments.
* @return never null
* @throws AssertionError when the expected penalty is not observed
*/
SingleConstraintAssertion indictsWithExactly(String message, Object... indictments);

/**
* Asserts that the {@link Constraint} being tested, given a set of facts, results in a specific penalty.
* <p>
Expand Down
136 changes: 72 additions & 64 deletions ...main/java/ai/timefold/solver/test/impl/score/stream/DefaultSingleConstraintAssertion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.BiPredicate;
import java.util.stream.Stream;

import ai.timefold.solver.core.api.score.Score;
import ai.timefold.solver.core.api.score.constraint.ConstraintMatchTotal;
Expand Down Expand Up @@ -52,13 +52,25 @@ public final class DefaultSingleConstraintAssertion<Solution_, Score_ extends Sc

@Override
public SingleConstraintAssertion justifiesWith(String message, ConstraintJustification... justifications) {
assertJustification(message, justifications);
assertJustification(message, false, justifications);
return this;
}

@Override
public SingleConstraintAssertion indictsWith(String message, Object... indictments) {
assertIndictments(message, indictments);
assertIndictments(message, false, indictments);
return this;
}

@Override
public SingleConstraintAssertion justifiesWithExactly(String message, ConstraintJustification... justifications) {
assertJustification(message, true, justifications);
return this;
}

@Override
public SingleConstraintAssertion indictsWithExactly(String message, Object... indictments) {
assertIndictments(message, true, indictments);
return this;
}

Expand Down Expand Up @@ -155,7 +167,7 @@ private void assertImpact(ScoreImpactType scoreImpactType, Number matchWeightTot
throw new AssertionError(assertionMessage);
}

private void assertJustification(String message, ConstraintJustification... justifications) {
private void assertJustification(String message, boolean completeValidation, ConstraintJustification... justifications) {
// Valid empty comparison
boolean emptyJustifications = justifications == null || justifications.length == 0;
if (emptyJustifications && justificationCollection.isEmpty()) {
Expand All @@ -165,39 +177,39 @@ private void assertJustification(String message, ConstraintJustification... just
// No justifications
if (emptyJustifications && !justificationCollection.isEmpty()) {
String assertionMessage = buildAssertionErrorMessage("Justification", constraint.getConstraintRef().constraintId(),
emptyList(), emptyList(), justificationCollection, message);
justificationCollection, emptyList(), emptyList(), justificationCollection, message);
throw new AssertionError(assertionMessage);
}

// Empty justifications
if (justificationCollection.isEmpty()) {
String assertionMessage = buildAssertionErrorMessage("Justification", constraint.getConstraintRef().constraintId(),
emptyList(), Arrays.asList(justifications), emptyList(), message);
emptyList(), Arrays.asList(justifications), Arrays.asList(justifications), emptyList(), message);
throw new AssertionError(assertionMessage);
}

List<Object> noneMatchedList = new LinkedList<>();
List<Object> invalidMatchList = new LinkedList<>();
List<Object> expectedNotFound = new ArrayList<>(justificationCollection.size());
for (Object justification : justifications) {
// No match
if (justificationCollection.stream().noneMatch(j -> justification.getClass().isAssignableFrom(j.getClass()))) {
noneMatchedList.add(justification);
continue;
}
// Test invalid match
if (justificationCollection.stream().noneMatch(justification::equals)) {
invalidMatchList.add(justification);
expectedNotFound.add(justification);
}
}
if (noneMatchedList.isEmpty() && invalidMatchList.isEmpty()) {
List<ConstraintJustification> unexpectedFound = emptyList();
if (completeValidation) {
unexpectedFound = justificationCollection.stream()
.filter(justification -> Stream.of(justifications).noneMatch(justification::equals))
.toList();
}
if (expectedNotFound.isEmpty() && unexpectedFound.isEmpty()) {
return;
}
String assertionMessage = buildAssertionErrorMessage("Justification", constraint.getConstraintRef().constraintId(),
noneMatchedList, invalidMatchList, justificationCollection, message);
unexpectedFound, expectedNotFound, Arrays.asList(justifications), justificationCollection, message);
throw new AssertionError(assertionMessage);
}

private void assertIndictments(String message, Object... indictments) {
private void assertIndictments(String message, boolean completeValidation, Object... indictments) {
boolean emptyIndictments = indictments == null || indictments.length == 0;
// Valid empty comparison
if (emptyIndictments && indictmentCollection.isEmpty()) {
Expand All @@ -208,35 +220,35 @@ private void assertIndictments(String message, Object... indictments) {
Collection<Object> indictmentObjectList = indictmentCollection.stream().map(Indictment::getIndictedObject).toList();
if (emptyIndictments && !indictmentObjectList.isEmpty()) {
String assertionMessage = buildAssertionErrorMessage("Indictment", constraint.getConstraintRef().constraintId(),
emptyList(), emptyList(), indictmentObjectList, message);
indictmentObjectList, emptyList(), emptyList(), indictmentObjectList, message);
throw new AssertionError(assertionMessage);
}

// Empty indictments
if (indictmentObjectList.isEmpty()) {
String assertionMessage = buildAssertionErrorMessage("Indictment", constraint.getConstraintRef().constraintId(),
emptyList(), Arrays.asList(indictments), emptyList(), message);
emptyList(), Arrays.asList(indictments), Arrays.asList(indictments), emptyList(), message);
throw new AssertionError(assertionMessage);
}

List<Object> noneMatchedList = new LinkedList<>();
List<Object> invalidMatchList = new LinkedList<>();
List<Object> expectedNotFound = new ArrayList<>(indictmentObjectList.size());
for (Object indictment : indictments) {
// No match
if (indictmentObjectList.stream().noneMatch(j -> indictment.getClass().isAssignableFrom(j.getClass()))) {
noneMatchedList.add(indictment);
continue;
}
// Test invalid match
if (indictmentObjectList.stream().noneMatch(indictment::equals)) {
invalidMatchList.add(indictment);
expectedNotFound.add(indictment);
}
}
if (noneMatchedList.isEmpty() && invalidMatchList.isEmpty()) {
List<Object> unexpectedFound = emptyList();
if (completeValidation) {
unexpectedFound = indictmentObjectList.stream()
.filter(indictment -> Arrays.stream(indictments).noneMatch(indictment::equals))
.toList();
}
if (expectedNotFound.isEmpty() && unexpectedFound.isEmpty()) {
return;
}
String assertionMessage = buildAssertionErrorMessage("Indictment", constraint.getConstraintRef().constraintId(),
noneMatchedList, invalidMatchList, indictmentObjectList, message);
unexpectedFound, expectedNotFound, Arrays.asList(indictments), indictmentObjectList, message);
throw new AssertionError(assertionMessage);
}

Expand Down Expand Up @@ -385,63 +397,59 @@ private String buildAssertionErrorMessage(ScoreImpactType impactType, String con
DefaultScoreExplanation.explainScore(score, constraintMatchTotalCollection, indictmentCollection));
}

private static String buildAssertionErrorMessage(String type, String constraintId, Collection<?> noneMatchedCollection,
Collection<?> invalidMatchedCollection, Collection<?> actualCollection,
private static String buildAssertionErrorMessage(String type, String constraintId, Collection<?> unexpectedFound,
Collection<?> expectedNotFound, Collection<?> expectedCollection, Collection<?> actualCollection,
String message) {
String expectation = message != null ? message : "Broken expectation.";
StringBuilder preformattedMessage = new StringBuilder("%s%n")
.append("%18s: %s%n");
List<Object> params = new LinkedList<>();
List<Object> params = new ArrayList<>();
params.add(expectation);
params.add(type);
params.add(constraintId);
if (!noneMatchedCollection.isEmpty()) {
preformattedMessage.append("%22s%n");
preformattedMessage.append("%24s%n");
params.add("No match:");
params.add("Expected:");
noneMatchedCollection.forEach(indictment -> {
preformattedMessage.append("%24s%n");
params.add("Expected:");
if (expectedCollection.isEmpty()) {
preformattedMessage.append("%26s%s%n");
params.add("");
params.add("No " + type);
} else {
expectedCollection.forEach(actual -> {
preformattedMessage.append("%26s%s%n");
params.add("");
params.add(indictment);
params.add(actual);
});
preformattedMessage.append("%24s%n");
params.add("Actual:");
}
preformattedMessage.append("%24s%n");
params.add("Actual:");
if (actualCollection.isEmpty()) {
preformattedMessage.append("%26s%s%n");
params.add("");
params.add("No " + type);
} else {
actualCollection.forEach(actual -> {
preformattedMessage.append("%26s%s%n");
params.add("");
params.add(actual);
});
}
if (!invalidMatchedCollection.isEmpty() || (noneMatchedCollection.isEmpty() && !actualCollection.isEmpty())) {
preformattedMessage.append("%22s%n");
if (!expectedNotFound.isEmpty()) {
preformattedMessage.append("%24s%n");
params.add("Invalid match:");
params.add("Expected:");
if (invalidMatchedCollection.isEmpty()) {
params.add("Expected but not found:");
expectedNotFound.forEach(indictment -> {
preformattedMessage.append("%26s%s%n");
params.add("");
params.add("No " + type);
} else {
invalidMatchedCollection.forEach(indictment -> {
preformattedMessage.append("%26s%s%n");
params.add("");
params.add(indictment);
});
}
params.add(indictment);
});
}
if (!unexpectedFound.isEmpty()) {
preformattedMessage.append("%24s%n");
params.add("Actual:");
if (actualCollection.isEmpty()) {
params.add("Unexpected but found:");
unexpectedFound.forEach(indictment -> {
preformattedMessage.append("%26s%s%n");
params.add("");
params.add("No " + type);
} else {
actualCollection.forEach(actual -> {
preformattedMessage.append("%26s%s%n");
params.add("");
params.add(actual);
});
}
params.add(indictment);
});
}
return String.format(preformattedMessage.toString(), params.toArray());
}
Expand Down
Loading

0 comments on commit a99e600

Please sign in to comment.