Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP]: Add asv benchmarking #207

Merged
Merged
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
423bb04
Inital benchmarking setup, suite
BradyPlanden Feb 19, 2024
694cd0c
Merge branch 'develop' into 179-add-airspeed-velocity-for-automated-b…
BradyPlanden Feb 19, 2024
9602190
Updt. asv config, changelog, parameterisaton benchmarks
BradyPlanden Feb 19, 2024
7d00ccf
Merge branch 'develop' into 179-add-airspeed-velocity-for-automated-b…
BradyPlanden Feb 21, 2024
7f7d6df
Add peridoci benchmark workflow and nox session
BradyPlanden Feb 21, 2024
8698a5c
updt results path for upload
BradyPlanden Feb 21, 2024
fc94e48
Increment to Python 3.12, fix typos, add `push` trigger to workflow
BradyPlanden Feb 24, 2024
1c63061
Updt. nox session name
BradyPlanden Feb 24, 2024
5540042
Updt. nox benchmarks session, tests for CI on benchmark workflow, rem…
BradyPlanden Feb 24, 2024
57d43ee
Merge branch 'develop' into 179-add-airspeed-velocity-for-automated-b…
BradyPlanden Feb 24, 2024
aceedc5
asv installation, calls
BradyPlanden Feb 24, 2024
4e3722f
Additional benchmarks, updt build wheels
BradyPlanden Mar 15, 2024
3315602
Merge branch 'develop' into 179-add-airspeed-velocity-for-automated-b…
BradyPlanden Mar 15, 2024
8f0daf1
updt permissions for deployment
BradyPlanden Mar 15, 2024
ebc7b62
add --global arg to git config
BradyPlanden Mar 15, 2024
aa9a395
Limit workflow to pybop repo
BradyPlanden Mar 16, 2024
8e6f609
Add random seed, updt branch target, increment python for publish job…
BradyPlanden Mar 16, 2024
7e530b0
Updt benchmark intial soc, add tracking of optimisation results, incr…
BradyPlanden Mar 17, 2024
2013161
Adds benchmark badge
BradyPlanden Mar 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Updt benchmark intial soc, add tracking of optimisation results, incr…
…ement default_max_unchanged_iterations for more stable default
  • Loading branch information
BradyPlanden committed Mar 17, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 7e530b06f581e01a4594edd952e3bde0baa909fa
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -313,3 +313,4 @@ $RECYCLE.BIN/

# Airspeed Velocity
*.asv/
results/
2 changes: 1 addition & 1 deletion benchmarks/benchmark_optim_construction.py
Original file line number Diff line number Diff line change
@@ -61,7 +61,7 @@ def setup(self, model, parameter_set, optimiser):

# Create fitting problem
problem = pybop.FittingProblem(
model=model_instance, dataset=dataset, parameters=parameters, init_soc=0.5
model=model_instance, dataset=dataset, parameters=parameters
)

# Create cost function
Original file line number Diff line number Diff line change
@@ -34,21 +34,26 @@ def setup(self, model, parameter_set, optimiser):
set_random_seed()

# Create model instance
model_instance = model(parameter_set=pybop.ParameterSet.pybamm(parameter_set))
params = pybop.ParameterSet.pybamm(parameter_set)
params.update(
{
"Negative electrode active material volume fraction": 0.63,
"Positive electrode active material volume fraction": 0.51,
}
)
model_instance = model(parameter_set=params)

# Define fitting parameters
parameters = [
pybop.Parameter(
"Negative electrode active material volume fraction",
prior=pybop.Gaussian(0.6, 0.03),
prior=pybop.Gaussian(0.55, 0.03),
bounds=[0.375, 0.7],
initial_value=0.63,
),
pybop.Parameter(
"Positive electrode active material volume fraction",
prior=pybop.Gaussian(0.5, 0.03),
bounds=[0.375, 0.625],
initial_value=0.51,
prior=pybop.Gaussian(0.55, 0.03),
bounds=[0.375, 0.7],
),
]

@@ -70,34 +75,58 @@ def setup(self, model, parameter_set, optimiser):
)

# Create fitting problem
problem = pybop.FittingProblem(
model=model_instance, dataset=dataset, parameters=parameters, init_soc=0.5
)
problem = pybop.FittingProblem(model_instance, parameters, dataset)

# Create cost function
cost = pybop.SumSquaredError(problem=problem)

# Create optimization instance
self.optim = pybop.Optimisation(cost, optimiser=optimiser)
if optimiser in [pybop.GradientDescent]:
self.optim.optimiser.set_learning_rate(
0.008
) # Compromise between stability & performance

def time_parameterisation(self, _model, _parameter_set, _optimiser):
def time_parameterisation(self, model, parameter_set, optimiser):
"""
Benchmark the parameterization process.
Benchmark the parameterization process. Optimiser options are left at high values
to ensure the threshold is met and the optimisation process is completed.

Args:
_model (pybop.Model): The model class being benchmarked (unused).
_parameter_set (str): The name of the parameter set being used (unused).
_optimiser (pybop.Optimiser): The optimizer class being used (unused).
model (pybop.Model): The model class being benchmarked (unused).
parameter_set (str): The name of the parameter set being used (unused).
optimiser (pybop.Optimiser): The optimizer class being used (unused).
"""
self.optim.run()
# Set optimizer options for consistent benchmarking
self.optim.set_max_unchanged_iterations(iterations=25, threshold=1e-5)
self.optim.set_max_iterations(250)
self.optim.set_min_iterations(2)
x, _ = self.optim.run()
return x

def track_results(self, model, parameter_set, optimiser):
"""
Track the results of the optimization.
Note: These results will be different than the time_parameterisation
as they are ran seperately. These results should be used to verify the
optimisation algorithm typically converges.

Args:
model (pybop.Model): The model class being benchmarked (unused).
parameter_set (str): The name of the parameter set being used (unused).
optimiser (pybop.Optimiser): The optimizer class being used (unused).
"""
x = self.time_parameterisation(model, parameter_set, optimiser)

return tuple(x)

def time_optimiser_ask(self, _model, _parameter_set, optimiser):
def time_optimiser_ask(self, model, parameter_set, optimiser):
"""
Benchmark the optimizer's ask method.

Args:
_model (pybop.Model): The model class being benchmarked (unused).
_parameter_set (str): The name of the parameter set being used (unused).
model (pybop.Model): The model class being benchmarked (unused).
parameter_set (str): The name of the parameter set being used (unused).
optimiser (pybop.Optimiser): The optimizer class being used.
"""
if optimiser not in [pybop.SciPyMinimize, pybop.SciPyDifferentialEvolution]:
4 changes: 2 additions & 2 deletions pybop/_optimisation.py
Original file line number Diff line number Diff line change
@@ -468,15 +468,15 @@ def set_min_iterations(self, iterations=2):
raise ValueError("Minimum number of iterations cannot be negative.")
self._min_iterations = iterations

def set_max_unchanged_iterations(self, iterations=5, threshold=1e-5):
def set_max_unchanged_iterations(self, iterations=15, threshold=1e-5):
"""
Set the maximum number of iterations without significant change as a stopping criterion.
Credit: PINTS

Parameters
----------
iterations : int, optional
The maximum number of unchanged iterations to run (default is 25).
The maximum number of unchanged iterations to run (default is 15).
Set to `None` to remove this stopping criterion.
threshold : float, optional
The minimum significant change in the objective function value that resets the unchanged iteration counter (default is 1e-5).
Loading