Benchmark suite and speed enhancements

.config/dotnet-tools.json

@@ -3,13 +3,13 @@
   "isRoot": true,
   "tools": {
     "fake-cli": {
-      "version": "5.20.4",
+      "version": "6.0.0",
       "commands": [
         "fake"
       ]
     },
     "paket": {
-      "version": "6.0.13",
+      "version": "8.0.3",
       "commands": [
         "paket"
       ]
@@ -19,6 +19,12 @@
       "commands": [
         "fsdocs"
       ]
+    },
+    "fantomas": {
+      "version": "6.3.1",
+      "commands": [
+        "fantomas"
+      ]
     }
   }
 }

.github/workflows/pr.yml

@@ -12,15 +12,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-20.04, macos-latest, windows-latest]
     env:
       RPROVIDER_LOG: rlog.txt
 
     steps:
     - uses: actions/checkout@master
     - uses: r-lib/actions/setup-r@v1
       name: Setup R - Windows / macOS
-      if: matrix.os != 'ubuntu-latest'
+      if: matrix.os != 'ubuntu-20.04'
       with:
           r-version: '4.0.2'
     - name: Set R_HOME environment variable
@@ -48,7 +48,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 5.0.x
+        dotnet-version: 6.0.x
     - name: Restore tool dependencies
       run: dotnet tool restore
     - name: Restore paket dependencies

.github/workflows/push.yml

@@ -11,13 +11,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
+        os: [macos-latest, ubuntu-20.04, windows-latest]
 
     steps:
     - uses: actions/checkout@master
     - uses: r-lib/actions/setup-r@v1
       name: Setup R - Windows / macOS
-      if: matrix.os != 'ubuntu-latest'
+      if: matrix.os != 'ubuntu-20.04'
       with:
           r-version: '4.0.2'
     - name: Set R_HOME environment variable
@@ -47,7 +47,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 5.0.x
+        dotnet-version: 6.0.x
     - name: Restore tool dependencies
       run: dotnet tool restore
     - name: Restore paket dependencies

README.MD

@@ -28,6 +28,11 @@ To get started with Bristlecone, see [the documentation website with tutorials,
 
 In addition, in the `samples` folder there are example scripts for simple to more complex models.
 
+#### Benchmarks
+
+A suite of benchmarks - covering optimisation methods and ecological time-series models - are included in the `test/Bristlecone.Benchmark` project. The most recent results of the benchmark tests [are here](benchmarks.md).
+
+
 Developers: Build Instructions
 ----------------
 

RELEASE_NOTES.MD

@@ -1,3 +1,7 @@
+#### 2.0.0-alpha2 - unreleased
+* FEATURE: Benchmark suite of common test functions for optimisation routines
+* FEATURE: Performance tuning of optimisers. For example, Filzbach 94% faster.
+
 #### 2.0.0-alpha2 - 2022-04-27
 * FEATURE: Add univariate gaussian -log likelihood function
 

benchmarks.md

@@ -0,0 +1,21 @@
+
+Bristlecone Benchmarks
+===
+
+## Optimisation Methods
+
+| Model name | Optimisation method | n runs | Target minimum | Best estimated minimum | Median estimated minimum | Sum distance from true parameters | Parameter estimates (individual) | True solutions | Milliseconds taken (median) |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| Ackley (2D) | amoeba single | 10 | 0.000 | 0.000 (0.000 -> min) | 4.731 | 1.958 | best 0.000 / median 0.000 (±7.136)<br>best -0.000 / median 0.000 (±9.188) | 0.000 / 0.000 | 288.5 |
+| Bukin Sixth | amoeba single | 10 | 0.000 | 0.002 (0.002 -> min) | 0.006 | 0.696 | best -10.214 / median -9.597 (±2.037)<br>best 1.043 / median 0.921 (±0.369) | -10.000 / 1.000 | 172 |
+| Holder Table | amoeba single | 10 | -19.209 | -19.209 (0.000 -> min) | -7.619 | 6.492 | best -8.055 / median -4.913 (±6.507)<br>best -9.665 / median 1.630 (±8.518) | 8.055 / 9.665<br>8.055 / -9.665<br>-8.055 / 9.665<br>-8.055 / -9.665 | 176.5 |
+| Cross in tray | amoeba single | 10 | -2.063 | -2.063 (0.000 -> min) | -2.063 | 0.001 | best 1.349 / median 1.349 (±3.721)<br>best -1.349 / median -1.349 (±2.326) | 1.349 / -1.349<br>1.349 / 1.349<br>-1.349 / 1.349<br>-1.349 / -1.349 | 192 |
+
+## Full Model Systems
+
+### Time-series models
+
+| Model name | Optimisation method | success % | n runs | Distance from minimum likelihood (median) | Distance from true parameter values | Milliseconds taken (median) |
+| --- | --- | --- | --- | --- | --- | --- |
+| predator-prey (with gaussian noise) | amoeba single | 100.000% | 10 | 2718.028 | [Δ] 0.053 best / median 0.265 (±1.094)<br>[α] 0.001 best / median 0.170 (±1.380)<br>[β] 0.088 best / median 0.314 (±0.479)<br>[γ] 0.026 best / median 0.271 (±0.227)<br>[ρ] 0.136 best / median 0.662 (±0.342)<br>[σ[x]] 0.007 best / median 0.209 (±0.249)<br>[σ[y]] 0.014 best / median 1.647 (±3755011282.332) | 311 |
+| predator-prey | amoeba single | 100.000% | 10 | 6.543 | [Δ] 0.013 best / median 0.114 (±0.167)<br>[α] 0.005 best / median 0.071 (±0.171)<br>[β] 0.001 best / median 0.120 (±0.097)<br>[γ] 0.027 best / median 0.171 (±0.103) | 557 |

bristlecone.sln

@@ -23,6 +23,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "project", "project", "{BF60
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{ED8079DD-2B06-4030-9F0F-DC548F98E1C4}"
+	ProjectSection(SolutionItems) = preProject
+	EndProjectSection
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Bristlecone.Tests", "tests\Bristlecone.Tests\Bristlecone.Tests.fsproj", "{D82667D7-EC01-4CD8-9D5E-C32232B62284}"
 EndProject
@@ -48,6 +50,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{E2553722-911
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Bristlecone.Charts.R", "src\Bristlecone.Charts.R\Bristlecone.Charts.R.fsproj", "{2F5FF7E7-E250-4B2D-A058-ABD0052D6D1B}"
 EndProject
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Bristlecone.Benchmark", "tests\Bristlecone.Benchmark\Bristlecone.Benchmark.fsproj", "{6FC53231-69B1-424B-82BC-11014A7168A1}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -70,6 +74,10 @@ Global
 		{2F5FF7E7-E250-4B2D-A058-ABD0052D6D1B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2F5FF7E7-E250-4B2D-A058-ABD0052D6D1B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2F5FF7E7-E250-4B2D-A058-ABD0052D6D1B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6FC53231-69B1-424B-82BC-11014A7168A1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6FC53231-69B1-424B-82BC-11014A7168A1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6FC53231-69B1-424B-82BC-11014A7168A1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6FC53231-69B1-424B-82BC-11014A7168A1}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -79,5 +87,6 @@ Global
 		{07A1121A-EB9F-4CFA-9D39-2ECC6BB58145} = {E2553722-911E-4F7A-A282-EBF14E807D3D}
 		{B7339FEC-0891-4DF8-8BB5-0B806A64F32F} = {E2553722-911E-4F7A-A282-EBF14E807D3D}
 		{2F5FF7E7-E250-4B2D-A058-ABD0052D6D1B} = {E2553722-911E-4F7A-A282-EBF14E807D3D}
+		{6FC53231-69B1-424B-82BC-11014A7168A1} = {ED8079DD-2B06-4030-9F0F-DC548F98E1C4}
 	EndGlobalSection
 EndGlobal

build.fsx

@@ -3,14 +3,7 @@
 // --------------------------------------------------------------------------------------
 
 #if FAKE
-#r "paket:
-nuget FAKE.Core.Target
-nuget FAKE.Core.ReleaseNotes
-nuget FAKE.DotNet.Cli
-nuget FAKE.DotNet.Fsi
-nuget FAKE.DotNet.AssemblyInfoFile
-nuget FAKE.Tools.Git
-nuget FAKE.DotNet.Testing.XUnit2"
+#r "paket: groupref Build //"
 #load "./.fake/build.fsx/intellisense.fsx"
 #else
 #r "nuget: FAKE.Core.Target"
@@ -21,7 +14,8 @@ nuget FAKE.DotNet.Testing.XUnit2"
 #r "nuget: FAKE.Tools.Git"
 #r "nuget: FAKE.DotNet.Testing.XUnit2"
 #r "nuget: System.Reactive"
-#r "nuget: MSBuild.StructuredLogger"
+#r "nuget: MSBuild.StructuredLogger, 2.1.820"
+
 let execContext = Fake.Core.Context.FakeExecutionContext.Create false "build.fsx" []
 Fake.Core.Context.setExecutionContext (Fake.Core.Context.RuntimeContext.Fake execContext)
 #endif

citest.fsx

@@ -1,4 +1,4 @@
-#r "nuget:RProvider,2.0.2"
+#r "nuget:RProvider,2.1.0"
 
 open RProvider
 open RProvider.``base``

global.json

@@ -0,0 +1,6 @@
+{
+    "sdk": {
+        "version": "6.0.0",
+        "rollForward": "latestFeature"
+    }
+}

paket.dependencies

@@ -1,17 +1,40 @@
-version 5.198.0
+version 8.0.3
 
 framework: auto-detect
 
 source https://nuget.org/api/v2
 storage: none
 
-nuget FSharp.Core 4.6.2
+nuget FSharp.Core ~> 5
 nuget FSharp.Data
 nuget MathNet.Numerics
 nuget MathNet.Numerics.FSharp
 nuget Accord.Statistics
-nuget RProvider 2.0.2
+nuget RProvider 2.1.0
 
-nuget Expecto
-nuget Expecto.BenchmarkDotNet
-nuget Expecto.FsCheck
+group Test
+    source https://nuget.org/api/v2
+    storage: none
+    nuget Expecto
+    nuget Expecto.BenchmarkDotNet
+    nuget Expecto.FsCheck
+    nuget Expecto.Hopac
+
+group Build
+  source https://api.nuget.org/v3/index.json
+  framework: net6.0
+  storage: none
+
+  nuget Microsoft.Build 17.3.2
+  nuget Microsoft.Build.Framework 17.3.2
+  nuget Microsoft.Build.Tasks.Core 17.3.2
+  nuget Microsoft.Build.Utilities.Core 17.3.2
+
+  nuget FAKE.Core.Target
+  nuget FAKE.Core.ReleaseNotes
+  nuget FAKE.DotNet.Cli
+  nuget FAKE.DotNet.Fsi
+  nuget FAKE.DotNet.AssemblyInfoFile
+  nuget FAKE.Tools.Git
+  nuget FAKE.DotNet.Testing.XUnit2
+  nuget MSBuild.StructuredLogger