feat: add new move count metric (#1072)

This pull request includes a new metric for counting the number of moves, which differs from the score count when using R&R moves. --------- Co-authored-by: Lukáš Petrovický <[email protected]>
TimefoldAI · Sep 26, 2024 · c027d7e · c027d7e
1 parent 141b70f
commit c027d7e
Show file tree

Hide file tree

Showing 83 changed files with 1,813 additions and 235 deletions.
diff --git a/...ark/src/main/java/ai/timefold/solver/benchmark/config/statistic/ProblemStatisticType.java b/...ark/src/main/java/ai/timefold/solver/benchmark/config/statistic/ProblemStatisticType.java
@@ -11,6 +11,8 @@
 import ai.timefold.solver.benchmark.impl.statistic.bestsolutionmutation.BestSolutionMutationProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.memoryuse.MemoryUseProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.movecountperstep.MoveCountPerStepProblemStatistic;
+import ai.timefold.solver.benchmark.impl.statistic.movecountpertype.MoveCountPerTypeProblemStatistic;
+import ai.timefold.solver.benchmark.impl.statistic.moveevaluationspeed.MoveEvaluationSpeedProblemStatisticTime;
 import ai.timefold.solver.benchmark.impl.statistic.scorecalculationspeed.ScoreCalculationSpeedProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.stepscore.StepScoreProblemStatistic;
 
@@ -19,8 +21,10 @@ public enum ProblemStatisticType implements StatisticType {
     BEST_SCORE,
     STEP_SCORE,
     SCORE_CALCULATION_SPEED,
+    MOVE_EVALUATION_SPEED,
     BEST_SOLUTION_MUTATION,
     MOVE_COUNT_PER_STEP,
+    MOVE_COUNT_PER_TYPE,
     MEMORY_USE;
 
     public ProblemStatistic buildProblemStatistic(ProblemBenchmarkResult problemBenchmarkResult) {
@@ -31,10 +35,14 @@ public ProblemStatistic buildProblemStatistic(ProblemBenchmarkResult problemBenc
                 return new StepScoreProblemStatistic(problemBenchmarkResult);
             case SCORE_CALCULATION_SPEED:
                 return new ScoreCalculationSpeedProblemStatistic(problemBenchmarkResult);
+            case MOVE_EVALUATION_SPEED:
+                return new MoveEvaluationSpeedProblemStatisticTime(problemBenchmarkResult);
             case BEST_SOLUTION_MUTATION:
                 return new BestSolutionMutationProblemStatistic(problemBenchmarkResult);
             case MOVE_COUNT_PER_STEP:
                 return new MoveCountPerStepProblemStatistic(problemBenchmarkResult);
+            case MOVE_COUNT_PER_TYPE:
+                return new MoveCountPerTypeProblemStatistic(problemBenchmarkResult);
             case MEMORY_USE:
                 return new MemoryUseProblemStatistic(problemBenchmarkResult);
             default:

diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/SolverBenchmarkFactory.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/SolverBenchmarkFactory.java
@@ -91,6 +91,8 @@ protected List<SolverMetric> getSolverMetrics(ProblemBenchmarksConfig config) {
                 .orElseGet(ProblemStatisticType::defaultList)) {
             if (problemStatisticType == ProblemStatisticType.SCORE_CALCULATION_SPEED) {
                 out.add(SolverMetric.SCORE_CALCULATION_COUNT);
+            } else if (problemStatisticType == ProblemStatisticType.MOVE_EVALUATION_SPEED) {
+                out.add(SolverMetric.MOVE_EVALUATION_COUNT);
             } else {
                 out.add(SolverMetric.valueOf(problemStatisticType.name()));
             }

diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/SubSingleBenchmarkRunner.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/SubSingleBenchmarkRunner.java
@@ -125,6 +125,7 @@ public SubSingleBenchmarkRunner<Solution_> call() {
             subSingleBenchmarkResult.setScore(solutionDescriptor.getScore(solution));
             subSingleBenchmarkResult.setTimeMillisSpent(timeMillisSpent);
             subSingleBenchmarkResult.setScoreCalculationCount(solverScope.getScoreCalculationCount());
+            subSingleBenchmarkResult.setMoveEvaluationCount(solverScope.getMoveEvaluationCount());
 
             SolutionManager<Solution_, ?> solutionManager = SolutionManager.create(solverFactory);
             boolean isConstraintMatchEnabled = solver.getSolverScope().getScoreDirector().isConstraintMatchEnabled();

diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/report/BenchmarkReport.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/report/BenchmarkReport.java
@@ -63,6 +63,7 @@ public static Configuration createFreeMarkerConfiguration() {
     private List<BarChart<Double>> winningScoreDifferenceSummaryChartList = null;
     private List<BarChart<Double>> worstScoreDifferencePercentageSummaryChartList = null;
     private LineChart<Long, Long> scoreCalculationSpeedSummaryChart;
+    private LineChart<Long, Long> moveEvaluationSpeedSummaryChart;
     private BarChart<Double> worstScoreCalculationSpeedDifferencePercentageSummaryChart = null;
     private BarChart<Long> timeSpentSummaryChart = null;
     private LineChart<Long, Long> timeSpentScalabilitySummaryChart = null;
@@ -142,6 +143,11 @@ public LineChart<Long, Long> getScoreCalculationSpeedSummaryChart() {
         return scoreCalculationSpeedSummaryChart;
     }
 
+    @SuppressWarnings("unused") // Used by FreeMarker.
+    public LineChart<Long, Long> getMoveEvaluationSpeedSummaryChart() {
+        return moveEvaluationSpeedSummaryChart;
+    }
+
     @SuppressWarnings("unused") // Used by FreeMarker.
     public BarChart<Double> getWorstScoreCalculationSpeedDifferencePercentageSummaryChart() {
         return worstScoreCalculationSpeedDifferencePercentageSummaryChart;
@@ -200,6 +206,7 @@ public void writeReport() {
         worstScoreDifferencePercentageSummaryChartList = createWorstScoreDifferencePercentageSummaryChart();
         bestScoreDistributionSummaryChartList = createBestScoreDistributionSummaryChart();
         scoreCalculationSpeedSummaryChart = createScoreCalculationSpeedSummaryChart();
+        moveEvaluationSpeedSummaryChart = createMoveEvaluationSpeedSummaryChart();
         worstScoreCalculationSpeedDifferencePercentageSummaryChart =
                 createWorstScoreCalculationSpeedDifferencePercentageSummaryChart();
         timeSpentSummaryChart = createTimeSpentSummaryChart();
@@ -239,6 +246,7 @@ public void writeReport() {
         chartsToWrite.addAll(worstScoreDifferencePercentageSummaryChartList);
         chartsToWrite.addAll(bestScoreDistributionSummaryChartList);
         chartsToWrite.add(scoreCalculationSpeedSummaryChart);
+        chartsToWrite.add(moveEvaluationSpeedSummaryChart);
         chartsToWrite.add(worstScoreCalculationSpeedDifferencePercentageSummaryChart);
         chartsToWrite.add(timeSpentSummaryChart);
         chartsToWrite.add(timeSpentScalabilitySummaryChart);
@@ -494,6 +502,12 @@ private LineChart<Long, Long> createScoreCalculationSpeedSummaryChart() {
                 "Score calculation speed per second", false);
     }
 
+    private LineChart<Long, Long> createMoveEvaluationSpeedSummaryChart() {
+        return createScalabilitySummaryChart(SingleBenchmarkResult::getMoveEvaluationSpeed,
+                "moveEvaluationSpeedSummaryChart", "Move evaluation speed summary (higher is better)",
+                "Move evaluation speed per second", false);
+    }
+
     private BarChart<Double> createWorstScoreCalculationSpeedDifferencePercentageSummaryChart() {
         return createSummaryBarChart(result -> result.getWorstScoreCalculationSpeedDifferencePercentage() * 100,
                 "worstScoreCalculationSpeedDifferencePercentageSummaryChart",

diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/ProblemBenchmarkResult.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/ProblemBenchmarkResult.java
@@ -33,6 +33,7 @@
 import ai.timefold.solver.benchmark.impl.statistic.bestsolutionmutation.BestSolutionMutationProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.memoryuse.MemoryUseProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.movecountperstep.MoveCountPerStepProblemStatistic;
+import ai.timefold.solver.benchmark.impl.statistic.moveevaluationspeed.MoveEvaluationSpeedProblemStatisticTime;
 import ai.timefold.solver.benchmark.impl.statistic.scorecalculationspeed.ScoreCalculationSpeedProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.stepscore.StepScoreProblemStatistic;
 import ai.timefold.solver.core.api.domain.solution.PlanningScore;
@@ -67,6 +68,7 @@ public class ProblemBenchmarkResult<Solution_> {
             @XmlElement(name = "bestScoreProblemStatistic", type = BestScoreProblemStatistic.class),
             @XmlElement(name = "stepScoreProblemStatistic", type = StepScoreProblemStatistic.class),
             @XmlElement(name = "scoreCalculationSpeedProblemStatistic", type = ScoreCalculationSpeedProblemStatistic.class),
+            @XmlElement(name = "moveEvaluationSpeedProblemStatistic", type = MoveEvaluationSpeedProblemStatisticTime.class),
             @XmlElement(name = "bestSolutionMutationProblemStatistic", type = BestSolutionMutationProblemStatistic.class),
             @XmlElement(name = "moveCountPerStepProblemStatistic", type = MoveCountPerStepProblemStatistic.class),
             @XmlElement(name = "memoryUseProblemStatistic", type = MemoryUseProblemStatistic.class),

diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SingleBenchmarkResult.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SingleBenchmarkResult.java
@@ -58,6 +58,7 @@ public class SingleBenchmarkResult implements BenchmarkResult {
     private double[] standardDeviationDoubles = null;
     private long timeMillisSpent = -1L;
     private long scoreCalculationCount = -1L;
+    private long moveEvaluationCount = -1L;
     private String scoreExplanationSummary = null;
 
     // ************************************************************************
@@ -151,6 +152,14 @@ public void setScoreCalculationCount(long scoreCalculationCount) {
         this.scoreCalculationCount = scoreCalculationCount;
     }
 
+    public long getMoveEvaluationCount() {
+        return moveEvaluationCount;
+    }
+
+    public void setMoveEvaluationCount(long moveEvaluationCount) {
+        this.moveEvaluationCount = moveEvaluationCount;
+    }
+
     @SuppressWarnings("unused") // Used by FreeMarker.
     public String getScoreExplanationSummary() {
         return scoreExplanationSummary;
@@ -258,6 +267,15 @@ public Long getScoreCalculationSpeed() {
         return scoreCalculationCount * 1000L / timeMillisSpent;
     }
 
+    public Long getMoveEvaluationSpeed() {
+        long timeSpent = this.timeMillisSpent;
+        if (timeSpent == 0L) {
+            // Avoid divide by zero exception on a fast CPU
+            timeSpent = 1L;
+        }
+        return moveEvaluationCount * 1000L / timeSpent;
+    }
+
     @SuppressWarnings("unused") // Used By FreeMarker.
     public boolean isWinner() {
         return ranking != null && ranking.intValue() == 0;
@@ -326,6 +344,7 @@ private void determineRepresentativeSubSingleBenchmarkResult() {
         usedMemoryAfterInputSolution = median.getUsedMemoryAfterInputSolution();
         timeMillisSpent = median.getTimeMillisSpent();
         scoreCalculationCount = median.getScoreCalculationCount();
+        moveEvaluationCount = median.getMoveEvaluationCount();
         scoreExplanationSummary = median.getScoreExplanationSummary();
     }
 

diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SolverBenchmarkResult.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SolverBenchmarkResult.java
@@ -56,6 +56,7 @@ public class SolverBenchmarkResult {
     private ScoreDifferencePercentage averageWorstScoreDifferencePercentage = null;
     // The average of the average is not just the overall average if the SingleBenchmarkResult's timeMillisSpent differ
     private Long averageScoreCalculationSpeed = null;
+    private Long averageMoveEvaluationSpeed = null;
     private Long averageTimeMillisSpent = null;
     private Double averageWorstScoreCalculationSpeedDifferencePercentage = null;
 
@@ -159,6 +160,11 @@ public Long getAverageScoreCalculationSpeed() {
         return averageScoreCalculationSpeed;
     }
 
+    @SuppressWarnings("unused") // Used by FreeMarker.
+    public Long getAverageMoveEvaluationSpeed() {
+        return averageMoveEvaluationSpeed;
+    }
+
     public Long getAverageTimeMillisSpent() {
         return averageTimeMillisSpent;
     }
@@ -282,6 +288,7 @@ protected void determineTotalsAndAverages() {
         totalWinningScoreDifference = null;
         ScoreDifferencePercentage totalWorstScoreDifferencePercentage = null;
         long totalScoreCalculationSpeed = 0L;
+        long totalMoveEvaluationSpeed = 0L;
         long totalTimeMillisSpent = 0L;
         double totalWorstScoreCalculationSpeedDifferencePercentage = 0.0;
         uninitializedSolutionCount = 0;
@@ -300,6 +307,7 @@ protected void determineTotalsAndAverages() {
                     totalWinningScoreDifference = singleBenchmarkResult.getWinningScoreDifference();
                     totalWorstScoreDifferencePercentage = singleBenchmarkResult.getWorstScoreDifferencePercentage();
                     totalScoreCalculationSpeed = singleBenchmarkResult.getScoreCalculationSpeed();
+                    totalMoveEvaluationSpeed = singleBenchmarkResult.getMoveEvaluationSpeed();
                     totalTimeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
                     totalWorstScoreCalculationSpeedDifferencePercentage = singleBenchmarkResult
                             .getWorstScoreCalculationSpeedDifferencePercentage();
@@ -311,6 +319,7 @@ protected void determineTotalsAndAverages() {
                     totalWorstScoreDifferencePercentage = totalWorstScoreDifferencePercentage.add(
                             singleBenchmarkResult.getWorstScoreDifferencePercentage());
                     totalScoreCalculationSpeed += singleBenchmarkResult.getScoreCalculationSpeed();
+                    totalMoveEvaluationSpeed += singleBenchmarkResult.getMoveEvaluationSpeed();
                     totalTimeMillisSpent += singleBenchmarkResult.getTimeMillisSpent();
                     totalWorstScoreCalculationSpeedDifferencePercentage += singleBenchmarkResult
                             .getWorstScoreCalculationSpeedDifferencePercentage();
@@ -322,6 +331,7 @@ protected void determineTotalsAndAverages() {
             averageScore = totalScore.divide(successCount);
             averageWorstScoreDifferencePercentage = totalWorstScoreDifferencePercentage.divide(successCount);
             averageScoreCalculationSpeed = totalScoreCalculationSpeed / successCount;
+            averageMoveEvaluationSpeed = totalMoveEvaluationSpeed / successCount;
             averageTimeMillisSpent = totalTimeMillisSpent / successCount;
             averageWorstScoreCalculationSpeedDifferencePercentage = totalWorstScoreCalculationSpeedDifferencePercentage
                     / successCount;

diff --git a/...mark/src/main/java/ai/timefold/solver/benchmark/impl/result/SubSingleBenchmarkResult.java b/...mark/src/main/java/ai/timefold/solver/benchmark/impl/result/SubSingleBenchmarkResult.java
@@ -1,5 +1,7 @@
 package ai.timefold.solver.benchmark.impl.result;
 
+import static ai.timefold.solver.core.impl.util.MathUtils.getSpeed;
+
 import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -58,6 +60,7 @@ public class SubSingleBenchmarkResult implements BenchmarkResult {
     private Score<?> score = null;
     private long timeMillisSpent = -1L;
     private long scoreCalculationCount = -1L;
+    private long moveEvaluationCount = -1L;
     private String scoreExplanationSummary = null;
 
     // ************************************************************************
@@ -161,6 +164,14 @@ public void setScoreCalculationCount(long scoreCalculationCount) {
         this.scoreCalculationCount = scoreCalculationCount;
     }
 
+    public long getMoveEvaluationCount() {
+        return moveEvaluationCount;
+    }
+
+    public void setMoveEvaluationCount(long moveEvaluationCount) {
+        this.moveEvaluationCount = moveEvaluationCount;
+    }
+
     public String getScoreExplanationSummary() {
         return scoreExplanationSummary;
     }
@@ -209,12 +220,12 @@ public boolean isScoreFeasible() {
 
     @SuppressWarnings("unused") // Used by FreeMarker.
     public Long getScoreCalculationSpeed() {
-        long timeMillisSpent = this.timeMillisSpent;
-        if (timeMillisSpent == 0L) {
-            // Avoid divide by zero exception on a fast CPU
-            timeMillisSpent = 1L;
-        }
-        return scoreCalculationCount * 1000L / timeMillisSpent;
+        return getSpeed(scoreCalculationCount, this.timeMillisSpent);
+    }
+
+    @SuppressWarnings("unused") // Used by FreeMarker.
+    public Long getMoveEvaluationSpeed() {
+        return getSpeed(moveEvaluationCount, this.timeMillisSpent);
     }
 
     @SuppressWarnings("unused") // Used by FreeMarker.
@@ -293,6 +304,7 @@ protected static SubSingleBenchmarkResult createMerge(
         newResult.score = oldResult.score;
         newResult.timeMillisSpent = oldResult.timeMillisSpent;
         newResult.scoreCalculationCount = oldResult.scoreCalculationCount;
+        newResult.moveEvaluationCount = oldResult.moveEvaluationCount;
 
         singleBenchmarkResult.getSubSingleBenchmarkResultList().add(newResult);
         return newResult;

diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/ProblemStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/ProblemStatistic.java
@@ -20,6 +20,7 @@
 import ai.timefold.solver.benchmark.impl.statistic.bestsolutionmutation.BestSolutionMutationProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.memoryuse.MemoryUseProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.movecountperstep.MoveCountPerStepProblemStatistic;
+import ai.timefold.solver.benchmark.impl.statistic.moveevaluationspeed.MoveEvaluationSpeedProblemStatisticTime;
 import ai.timefold.solver.benchmark.impl.statistic.scorecalculationspeed.ScoreCalculationSpeedProblemStatistic;
 import ai.timefold.solver.benchmark.impl.statistic.stepscore.StepScoreProblemStatistic;
 
@@ -31,6 +32,7 @@
         BestScoreProblemStatistic.class,
         StepScoreProblemStatistic.class,
         ScoreCalculationSpeedProblemStatistic.class,
+        MoveEvaluationSpeedProblemStatisticTime.class,
         BestSolutionMutationProblemStatistic.class,
         MoveCountPerStepProblemStatistic.class,
         MemoryUseProblemStatistic.class

diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java
@@ -9,6 +9,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.ObjLongConsumer;
 import java.util.stream.Collectors;
 
 import ai.timefold.solver.core.api.score.Score;
@@ -32,6 +33,7 @@ public class StatisticRegistry<Solution_> extends SimpleMeterRegistry
     private static final String CONSTRAINT_PACKAGE_TAG = "constraint.package";
     private static final String CONSTRAINT_NAME_TAG = "constraint.name";
 
+    List<Consumer<SolverScope<Solution_>>> solverMeterListenerList = new ArrayList<>();
     List<BiConsumer<Long, AbstractStepScope<Solution_>>> stepMeterListenerList = new ArrayList<>();
     List<BiConsumer<Long, AbstractStepScope<Solution_>>> bestSolutionMeterListenerList = new ArrayList<>();
     AbstractStepScope<Solution_> bestSolutionStepScope = null;
@@ -77,6 +79,10 @@ public void addListener(SolverMetric metric, BiConsumer<Long, AbstractStepScope<
         }
     }
 
+    public void addListener(Consumer<SolverScope<Solution_>> listener) {
+        solverMeterListenerList.add(listener);
+    }
+
     public Set<Meter.Id> getMeterIds(SolverMetric metric, Tags runId) {
         return Search.in(this).name(name -> name.startsWith(metric.getMeterId())).tags(runId)
                 .meters().stream().map(Meter::getId)
@@ -131,6 +137,16 @@ public void getGaugeValue(String meterId, Tags runId, Consumer<Number> gaugeCons
         }
     }
 
+    public void extractMoveCountPerType(SolverScope<Solution_> solverScope, ObjLongConsumer<String> gaugeConsumer) {
+        solverScope.getMoveCountTypes().forEach(type -> {
+            var gauge = this.find(SolverMetric.MOVE_COUNT_PER_TYPE.getMeterId() + "." + type)
+                    .tags(solverScope.getMonitoringTags()).gauge();
+            if (gauge != null) {
+                gaugeConsumer.accept(type, (long) gauge.value());
+            }
+        });
+    }
+
     @Override
     protected TimeUnit getBaseTimeUnit() {
         return TimeUnit.MILLISECONDS;
@@ -181,5 +197,6 @@ public void solvingEnded(SolverScope<Solution_> solverScope) {
                     .forEach(listener -> listener.accept(bestSolutionChangedTimestamp, bestSolutionStepScope));
             lastStepImprovedSolution = false;
         }
+        solverMeterListenerList.forEach(listener -> listener.accept(solverScope));
     }
 }