From 5aca056cf3a8708eb4ee3fda9246b1358adaf9cf Mon Sep 17 00:00:00 2001
From: ST John <st--@users.noreply.github.com>
Date: Wed, 30 Jun 2021 17:13:02 +0300
Subject: [PATCH 01/41] initial version of training kernel parameters example
 from st/examples (#234)

---
 .../train-kernel-parameters/Manifest.toml     | 1088 +++++++++++++++++
 examples/train-kernel-parameters/Project.toml |   17 +
 examples/train-kernel-parameters/script.jl    |   84 ++
 3 files changed, 1189 insertions(+)
 create mode 100644 examples/train-kernel-parameters/Manifest.toml
 create mode 100644 examples/train-kernel-parameters/Project.toml
 create mode 100644 examples/train-kernel-parameters/script.jl

diff --git a/examples/train-kernel-parameters/Manifest.toml b/examples/train-kernel-parameters/Manifest.toml
new file mode 100644
index 000000000..7a3bb3837
--- /dev/null
+++ b/examples/train-kernel-parameters/Manifest.toml
@@ -0,0 +1,1088 @@
+# This file is machine-generated - editing it directly is not advised
+
+[[AbstractFFTs]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0"
+uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
+version = "1.0.1"
+
+[[AbstractTrees]]
+git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5"
+uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
+version = "0.3.4"
+
+[[Adapt]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7"
+uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
+version = "3.3.1"
+
+[[ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+
+[[Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
+[[BFloat16s]]
+deps = ["LinearAlgebra", "Test"]
+git-tree-sha1 = "4af69e205efc343068dc8722b8dfec1ade89254a"
+uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b"
+version = "0.1.0"
+
+[[Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[Bzip2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "c3598e525718abcc440f69cc6d5f60dda0a1b61e"
+uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
+version = "1.0.6+5"
+
+[[CEnum]]
+git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
+uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
+version = "0.4.1"
+
+[[CUDA]]
+deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"]
+git-tree-sha1 = "82b2811f5888465d96b38c7bb12d8fb9c25838e1"
+uuid = "052768ef-5323-5732-b1bb-66c8b64840ba"
+version = "3.3.1"
+
+[[Cairo_jll]]
+deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
+git-tree-sha1 = "e2f47f6d8337369411569fd45ae5753ca10394c6"
+uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
+version = "1.16.0+6"
+
+[[ChainRules]]
+deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Statistics"]
+git-tree-sha1 = "e7e3200bd24b77bcc849e6616f7c2f0d45d70f5b"
+uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2"
+version = "0.8.17"
+
+[[ChainRulesCore]]
+deps = ["Compat", "LinearAlgebra", "SparseArrays"]
+git-tree-sha1 = "be770c08881f7bb928dfd86d1ba83798f76cf62a"
+uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+version = "0.10.9"
+
+[[CodecZlib]]
+deps = ["TranscodingStreams", "Zlib_jll"]
+git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
+uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
+version = "0.7.0"
+
+[[ColorSchemes]]
+deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random", "StaticArrays"]
+git-tree-sha1 = "c8fd01e4b736013bc61b704871d20503b33ea402"
+uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
+version = "3.12.1"
+
+[[ColorTypes]]
+deps = ["FixedPointNumbers", "Random"]
+git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597"
+uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
+version = "0.11.0"
+
+[[Colors]]
+deps = ["ColorTypes", "FixedPointNumbers", "Reexport"]
+git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40"
+uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
+version = "0.12.8"
+
+[[CommonSubexpressions]]
+deps = ["MacroTools", "Test"]
+git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7"
+uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
+version = "0.3.0"
+
+[[Compat]]
+deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
+git-tree-sha1 = "dc7dedc2c2aa9faf59a55c622760a25cbefbe941"
+uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
+version = "3.31.0"
+
+[[CompilerSupportLibraries_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
+
+[[CompositionsBase]]
+git-tree-sha1 = "f3955eb38944e5dd0fabf8ca1e267d94941d34a5"
+uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
+version = "0.1.0"
+
+[[Contour]]
+deps = ["StaticArrays"]
+git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7"
+uuid = "d38c429a-6771-53c6-b99e-75d170b6e991"
+version = "0.5.7"
+
+[[DataAPI]]
+git-tree-sha1 = "ee400abb2298bd13bfc3df1c412ed228061a2385"
+uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+version = "1.7.0"
+
+[[DataStructures]]
+deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
+git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677"
+uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+version = "0.18.9"
+
+[[DataValueInterfaces]]
+git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
+uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
+version = "1.0.0"
+
+[[Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[DelimitedFiles]]
+deps = ["Mmap"]
+uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+
+[[DiffResults]]
+deps = ["StaticArrays"]
+git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805"
+uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
+version = "1.0.3"
+
+[[DiffRules]]
+deps = ["NaNMath", "Random", "SpecialFunctions"]
+git-tree-sha1 = "214c3fcac57755cfda163d91c58893a8723f93e9"
+uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
+version = "1.0.2"
+
+[[Distances]]
+deps = ["LinearAlgebra", "Statistics", "StatsAPI"]
+git-tree-sha1 = "abe4ad222b26af3337262b8afb28fab8d215e9f8"
+uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
+version = "0.10.3"
+
+[[Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+
+[[Distributions]]
+deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"]
+git-tree-sha1 = "2733323e5c02a9d7f48e7a3c4bc98d764fb704da"
+uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
+version = "0.25.6"
+
+[[DocStringExtensions]]
+deps = ["LibGit2"]
+git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
+uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+version = "0.8.5"
+
+[[Downloads]]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+
+[[EarCut_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "92d8f9f208637e8d2d28c664051a00569c01493d"
+uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5"
+version = "2.1.5+1"
+
+[[Expat_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "b3bfd02e98aedfa5cf885665493c5598c350cd2f"
+uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
+version = "2.2.10+0"
+
+[[ExprTools]]
+git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e"
+uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
+version = "0.1.3"
+
+[[FFMPEG]]
+deps = ["FFMPEG_jll"]
+git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8"
+uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a"
+version = "0.4.1"
+
+[[FFMPEG_jll]]
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "LibVPX_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
+git-tree-sha1 = "3cc57ad0a213808473eafef4845a74766242e05f"
+uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
+version = "4.3.1+4"
+
+[[FillArrays]]
+deps = ["LinearAlgebra", "Random", "SparseArrays"]
+git-tree-sha1 = "a603e79b71bb3c1efdb58f0ee32286efe2d1a255"
+uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
+version = "0.11.8"
+
+[[FixedPointNumbers]]
+deps = ["Statistics"]
+git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc"
+uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
+version = "0.8.4"
+
+[[Flux]]
+deps = ["AbstractTrees", "Adapt", "CUDA", "CodecZlib", "Colors", "DelimitedFiles", "Functors", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "NNlibCUDA", "Pkg", "Printf", "Random", "Reexport", "SHA", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"]
+git-tree-sha1 = "0b3c6d0ce57d3b793eabd346ccc8f605035ef079"
+uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+version = "0.12.4"
+
+[[Fontconfig_jll]]
+deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "35895cf184ceaab11fd778b4590144034a167a2f"
+uuid = "a3f928ae-7b40-5064-980b-68af3947d34b"
+version = "2.13.1+14"
+
+[[Formatting]]
+deps = ["Printf"]
+git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
+uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
+version = "0.4.2"
+
+[[ForwardDiff]]
+deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
+git-tree-sha1 = "e2af66012e08966366a43251e1fd421522908be6"
+uuid = "f6369f11-7733-5829-9624-2563aa707210"
+version = "0.10.18"
+
+[[FreeType2_jll]]
+deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "cbd58c9deb1d304f5a245a0b7eb841a2560cfec6"
+uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7"
+version = "2.10.1+5"
+
+[[FriBidi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91"
+uuid = "559328eb-81f9-559d-9380-de523a88c83c"
+version = "1.0.10+0"
+
+[[Functors]]
+deps = ["MacroTools"]
+git-tree-sha1 = "a7bb2af991c43dcf5c3455d276dd83976799634f"
+uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
+version = "0.2.1"
+
+[[GLFW_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"]
+git-tree-sha1 = "dba1e8614e98949abfa60480b13653813d8f0157"
+uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89"
+version = "3.3.5+0"
+
+[[GPUArrays]]
+deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"]
+git-tree-sha1 = "ececbf05f8904c92814bdbd0aafd5540b0bf2e9a"
+uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
+version = "7.0.1"
+
+[[GPUCompiler]]
+deps = ["DataStructures", "ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "TimerOutputs", "UUIDs"]
+git-tree-sha1 = "222c6cdb888ec24795936d6829aa978691def60e"
+uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
+version = "0.12.3"
+
+[[GR]]
+deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"]
+git-tree-sha1 = "b83e3125048a9c3158cbb7ca423790c7b1b57bea"
+uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
+version = "0.57.5"
+
+[[GR_jll]]
+deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"]
+git-tree-sha1 = "e14907859a1d3aee73a019e7b3c98e9e7b8b5b3e"
+uuid = "d2c73de3-f751-5644-a686-071e5b155ba9"
+version = "0.57.3+0"
+
+[[GeometryBasics]]
+deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"]
+git-tree-sha1 = "15ff9a14b9e1218958d3530cc288cf31465d9ae2"
+uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
+version = "0.3.13"
+
+[[Gettext_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"]
+git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046"
+uuid = "78b55507-aeef-58d4-861c-77aaff3498b1"
+version = "0.21.0+0"
+
+[[Glib_jll]]
+deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "47ce50b742921377301e15005c96e979574e130b"
+uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
+version = "2.68.1+0"
+
+[[Grisu]]
+git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2"
+uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe"
+version = "1.0.2"
+
+[[HTTP]]
+deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
+git-tree-sha1 = "99c6bc381e98c5331f34b9ce949aebea209bd0db"
+uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+version = "0.9.11"
+
+[[IOCapture]]
+deps = ["Logging", "Random"]
+git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
+uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
+version = "0.2.2"
+
+[[IRTools]]
+deps = ["InteractiveUtils", "MacroTools", "Test"]
+git-tree-sha1 = "95215cd0076a150ef46ff7928892bc341864c73c"
+uuid = "7869d1d1-7146-5819-86e3-90919afe41df"
+version = "0.4.3"
+
+[[IniFile]]
+deps = ["Test"]
+git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
+uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
+version = "0.5.0"
+
+[[InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[IterTools]]
+git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18"
+uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
+version = "1.3.0"
+
+[[IteratorInterfaceExtensions]]
+git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
+uuid = "82899510-4779-5014-852e-03e436cf321d"
+version = "1.0.0"
+
+[[JLLWrappers]]
+deps = ["Preferences"]
+git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
+uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
+version = "1.3.0"
+
+[[JSON]]
+deps = ["Dates", "Mmap", "Parsers", "Unicode"]
+git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.21.1"
+
+[[JpegTurbo_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "d735490ac75c5cb9f1b00d8b5509c11984dc6943"
+uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8"
+version = "2.1.0+0"
+
+[[Juno]]
+deps = ["Base64", "Logging", "Media", "Profile"]
+git-tree-sha1 = "07cb43290a840908a771552911a6274bc6c072c7"
+uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d"
+version = "0.8.4"
+
+[[KernelFunctions]]
+deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "LinearAlgebra", "Random", "Requires", "SpecialFunctions", "StatsBase", "StatsFuns", "TensorCore", "Test", "ZygoteRules"]
+git-tree-sha1 = "e8b5ba31b6d18695fd46bfcd8557682839023195"
+uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
+version = "0.10.6"
+
+[[LAME_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c"
+uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d"
+version = "3.100.1+0"
+
+[[LLVM]]
+deps = ["CEnum", "Libdl", "Printf", "Unicode"]
+git-tree-sha1 = "f57ac3fd2045b50d3db081663837ac5b4096947e"
+uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
+version = "3.9.0"
+
+[[LZO_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6"
+uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
+version = "2.10.1+0"
+
+[[LaTeXStrings]]
+git-tree-sha1 = "c7f1c695e06c01b95a67f0cd1d34994f3e7db104"
+uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
+version = "1.2.1"
+
+[[Latexify]]
+deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"]
+git-tree-sha1 = "a4b12a1bd2ebade87891ab7e36fdbce582301a92"
+uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
+version = "0.15.6"
+
+[[LazyArtifacts]]
+deps = ["Artifacts", "Pkg"]
+uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
+
+[[LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+
+[[LibCURL_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+
+[[LibGit2]]
+deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+
+[[LibVPX_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "12ee7e23fa4d18361e7c2cde8f8337d4c3101bc7"
+uuid = "dd192d2f-8180-539f-9fb4-cc70b1dcf69a"
+version = "1.10.0+0"
+
+[[Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[Libffi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "761a393aeccd6aa92ec3515e428c26bf99575b3b"
+uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490"
+version = "3.2.2+0"
+
+[[Libgcrypt_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"]
+git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae"
+uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4"
+version = "1.8.7+0"
+
+[[Libglvnd_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_libXext_jll"]
+git-tree-sha1 = "7739f837d6447403596a75d19ed01fd08d6f56bf"
+uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29"
+version = "1.3.0+3"
+
+[[Libgpg_error_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9"
+uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8"
+version = "1.42.0+0"
+
+[[Libiconv_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778"
+uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
+version = "1.16.1+1"
+
+[[Libmount_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73"
+uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9"
+version = "2.35.0+0"
+
+[[Libtiff_jll]]
+deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"]
+git-tree-sha1 = "340e257aada13f95f98ee352d316c3bed37c8ab9"
+uuid = "89763e89-9b03-5906-acba-b20f662cd828"
+version = "4.3.0+0"
+
+[[Libuuid_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066"
+uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700"
+version = "2.36.0+0"
+
+[[LinearAlgebra]]
+deps = ["Libdl"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
+[[Literate]]
+deps = ["Base64", "IOCapture", "JSON", "REPL"]
+git-tree-sha1 = "501a1a74a0c825037860d36d87d703e987d39dbc"
+uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+version = "2.8.1"
+
+[[LogExpFunctions]]
+deps = ["DocStringExtensions", "LinearAlgebra"]
+git-tree-sha1 = "1ba664552f1ef15325e68dc4c05c3ef8c2d5d885"
+uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
+version = "0.2.4"
+
+[[Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[MacroTools]]
+deps = ["Markdown", "Random"]
+git-tree-sha1 = "6a8a2a625ab0dea913aba95c11370589e0239ff0"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.6"
+
+[[Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[MbedTLS]]
+deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"]
+git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe"
+uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
+version = "1.0.3"
+
+[[MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+
+[[Measures]]
+git-tree-sha1 = "e498ddeee6f9fdb4551ce855a46f54dbd900245f"
+uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e"
+version = "0.3.1"
+
+[[Media]]
+deps = ["MacroTools", "Test"]
+git-tree-sha1 = "75a54abd10709c01f1b86b84ec225d26e840ed58"
+uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27"
+version = "0.5.0"
+
+[[Missings]]
+deps = ["DataAPI"]
+git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7"
+uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
+version = "1.0.0"
+
+[[Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"
+
+[[MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+
+[[NNlib]]
+deps = ["Adapt", "ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"]
+git-tree-sha1 = "7461639cef384a2ad058005b49e32b318d844343"
+uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+version = "0.7.22"
+
+[[NNlibCUDA]]
+deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"]
+git-tree-sha1 = "c2f2f152340022ce44c6ac97e05c62a6de59c373"
+uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d"
+version = "0.1.4"
+
+[[NaNMath]]
+git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb"
+uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
+version = "0.3.5"
+
+[[NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+
+[[Ogg_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f"
+uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051"
+version = "1.3.5+0"
+
+[[OpenSSL_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a"
+uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
+version = "1.1.10+0"
+
+[[OpenSpecFun_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
+uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
+version = "0.5.5+0"
+
+[[Opus_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720"
+uuid = "91d4177d-7536-5919-b921-800302f37372"
+version = "1.3.2+0"
+
+[[OrderedCollections]]
+git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.4.1"
+
+[[PCRE_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "b2a7af664e098055a7529ad1a900ded962bca488"
+uuid = "2f80f16e-611a-54ab-bc61-aa92de5b98fc"
+version = "8.44.0+0"
+
+[[PDMats]]
+deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
+git-tree-sha1 = "4dd403333bcf0909341cfe57ec115152f937d7d8"
+uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
+version = "0.11.1"
+
+[[Parsers]]
+deps = ["Dates"]
+git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc"
+uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+version = "1.1.0"
+
+[[Pixman_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "b4f5d02549a10e20780a24fce72bea96b6329e29"
+uuid = "30392449-352a-5448-841d-b1acce4e97dc"
+version = "0.40.1+0"
+
+[[Pkg]]
+deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+
+[[PlotThemes]]
+deps = ["PlotUtils", "Requires", "Statistics"]
+git-tree-sha1 = "a3a964ce9dc7898193536002a6dd892b1b5a6f1d"
+uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a"
+version = "2.0.1"
+
+[[PlotUtils]]
+deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"]
+git-tree-sha1 = "ae9a295ac761f64d8c2ec7f9f24d21eb4ffba34d"
+uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043"
+version = "1.0.10"
+
+[[Plots]]
+deps = ["Base64", "Contour", "Dates", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"]
+git-tree-sha1 = "df601eed7c9637235a26b26f9f648deccd277178"
+uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+version = "1.16.7"
+
+[[Preferences]]
+deps = ["TOML"]
+git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+version = "1.2.2"
+
+[[Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[Profile]]
+deps = ["Printf"]
+uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
+
+[[Qt5Base_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"]
+git-tree-sha1 = "ad368663a5e20dbb8d6dc2fddeefe4dae0781ae8"
+uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1"
+version = "5.15.3+0"
+
+[[QuadGK]]
+deps = ["DataStructures", "LinearAlgebra"]
+git-tree-sha1 = "12fbe86da16df6679be7521dfb39fbc861e1dc7b"
+uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
+version = "2.4.1"
+
+[[REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[Random]]
+deps = ["Serialization"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[Random123]]
+deps = ["Libdl", "Random", "RandomNumbers"]
+git-tree-sha1 = "0e8b146557ad1c6deb1367655e052276690e71a3"
+uuid = "74087812-796a-5b5d-8853-05524746bad3"
+version = "1.4.2"
+
+[[RandomNumbers]]
+deps = ["Random", "Requires"]
+git-tree-sha1 = "441e6fc35597524ada7f85e13df1f4e10137d16f"
+uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143"
+version = "1.4.0"
+
+[[RecipesBase]]
+git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae"
+uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
+version = "1.1.1"
+
+[[RecipesPipeline]]
+deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"]
+git-tree-sha1 = "9b8e57e3cca8828a1bc759840bfe48d64db9abfb"
+uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c"
+version = "0.3.3"
+
+[[Reexport]]
+git-tree-sha1 = "5f6c21241f0f655da3952fd60aa18477cf96c220"
+uuid = "189a3867-3050-52da-a836-e630ba90ab69"
+version = "1.1.0"
+
+[[Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.1.3"
+
+[[Rmath]]
+deps = ["Random", "Rmath_jll"]
+git-tree-sha1 = "bf3188feca147ce108c76ad82c2792c57abe7b1f"
+uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
+version = "0.7.0"
+
+[[Rmath_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "68db32dff12bb6127bac73c209881191bf0efbb7"
+uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
+version = "0.3.0+0"
+
+[[SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+
+[[Scratch]]
+deps = ["Dates"]
+git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda"
+uuid = "6c6a2e73-6563-6170-7368-637461726353"
+version = "1.1.0"
+
+[[Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[SharedArrays]]
+deps = ["Distributed", "Mmap", "Random", "Serialization"]
+uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
+
+[[Showoff]]
+deps = ["Dates", "Grisu"]
+git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de"
+uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f"
+version = "1.0.3"
+
+[[Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[SortingAlgorithms]]
+deps = ["DataStructures"]
+git-tree-sha1 = "2ec1962eba973f383239da22e75218565c390a96"
+uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
+version = "1.0.0"
+
+[[SparseArrays]]
+deps = ["LinearAlgebra", "Random"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+
+[[SpecialFunctions]]
+deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"]
+git-tree-sha1 = "a50550fa3164a8c46747e62063b4d774ac1bcf49"
+uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
+version = "1.5.1"
+
+[[StaticArrays]]
+deps = ["LinearAlgebra", "Random", "Statistics"]
+git-tree-sha1 = "745914ebcd610da69f3cb6bf76cb7bb83dcb8c9a"
+uuid = "90137ffa-7385-5640-81b9-e52037218182"
+version = "1.2.4"
+
+[[Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[[StatsAPI]]
+git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
+uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
+version = "1.0.0"
+
+[[StatsBase]]
+deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "2f6792d523d7448bbe2fec99eca9218f06cc746d"
+uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+version = "0.33.8"
+
+[[StatsFuns]]
+deps = ["LogExpFunctions", "Rmath", "SpecialFunctions"]
+git-tree-sha1 = "30cd8c360c54081f806b1ee14d2eecbef3c04c49"
+uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
+version = "0.9.8"
+
+[[StructArrays]]
+deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"]
+git-tree-sha1 = "000e168f5cc9aded17b6999a560b7c11dda69095"
+uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
+version = "0.6.0"
+
+[[SuiteSparse]]
+deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
+uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
+
+[[TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+
+[[TableTraits]]
+deps = ["IteratorInterfaceExtensions"]
+git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
+uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
+version = "1.0.1"
+
+[[Tables]]
+deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
+git-tree-sha1 = "8ed4a3ea724dac32670b062be3ef1c1de6773ae8"
+uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+version = "1.4.4"
+
+[[Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+
+[[TensorCore]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6"
+uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50"
+version = "0.1.1"
+
+[[Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[TimerOutputs]]
+deps = ["ExprTools", "Printf"]
+git-tree-sha1 = "9f494bc54b4c31404a9eff449235836615929de1"
+uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
+version = "0.5.10"
+
+[[TranscodingStreams]]
+deps = ["Random", "Test"]
+git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
+uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
+version = "0.9.5"
+
+[[URIs]]
+git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355"
+uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
+version = "1.3.0"
+
+[[UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[Wayland_jll]]
+deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"]
+git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23"
+uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89"
+version = "1.19.0+0"
+
+[[Wayland_protocols_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll"]
+git-tree-sha1 = "2839f1c1296940218e35df0bbb220f2a79686670"
+uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91"
+version = "1.18.0+4"
+
+[[XML2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a"
+uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
+version = "2.9.12+0"
+
+[[XSLT_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"]
+git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a"
+uuid = "aed1982a-8fda-507f-9586-7b0439959a61"
+version = "1.1.34+0"
+
+[[Xorg_libX11_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll", "Xorg_xtrans_jll"]
+git-tree-sha1 = "5be649d550f3f4b95308bf0183b82e2582876527"
+uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc"
+version = "1.6.9+4"
+
+[[Xorg_libXau_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "4e490d5c960c314f33885790ed410ff3a94ce67e"
+uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec"
+version = "1.0.9+4"
+
+[[Xorg_libXcursor_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"]
+git-tree-sha1 = "12e0eb3bc634fa2080c1c37fccf56f7c22989afd"
+uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724"
+version = "1.2.0+4"
+
+[[Xorg_libXdmcp_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "4fe47bd2247248125c428978740e18a681372dd4"
+uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05"
+version = "1.1.3+4"
+
+[[Xorg_libXext_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3"
+uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3"
+version = "1.3.4+4"
+
+[[Xorg_libXfixes_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "0e0dc7431e7a0587559f9294aeec269471c991a4"
+uuid = "d091e8ba-531a-589c-9de9-94069b037ed8"
+version = "5.0.3+4"
+
+[[Xorg_libXi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll"]
+git-tree-sha1 = "89b52bc2160aadc84d707093930ef0bffa641246"
+uuid = "a51aa0fd-4e3c-5386-b890-e753decda492"
+version = "1.7.10+4"
+
+[[Xorg_libXinerama_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll"]
+git-tree-sha1 = "26be8b1c342929259317d8b9f7b53bf2bb73b123"
+uuid = "d1454406-59df-5ea1-beac-c340f2130bc3"
+version = "1.1.4+4"
+
+[[Xorg_libXrandr_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll"]
+git-tree-sha1 = "34cea83cb726fb58f325887bf0612c6b3fb17631"
+uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484"
+version = "1.5.2+4"
+
+[[Xorg_libXrender_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96"
+uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa"
+version = "0.9.10+4"
+
+[[Xorg_libpthread_stubs_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "6783737e45d3c59a4a4c4091f5f88cdcf0908cbb"
+uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74"
+version = "0.1.0+3"
+
+[[Xorg_libxcb_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"]
+git-tree-sha1 = "daf17f441228e7a3833846cd048892861cff16d6"
+uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b"
+version = "1.13.0+3"
+
+[[Xorg_libxkbfile_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "926af861744212db0eb001d9e40b5d16292080b2"
+uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a"
+version = "1.1.0+4"
+
+[[Xorg_xcb_util_image_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97"
+uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b"
+version = "0.4.0+1"
+
+[[Xorg_xcb_util_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"]
+git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1"
+uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5"
+version = "0.4.0+1"
+
+[[Xorg_xcb_util_keysyms_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00"
+uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7"
+version = "0.4.0+1"
+
+[[Xorg_xcb_util_renderutil_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e"
+uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e"
+version = "0.3.9+1"
+
+[[Xorg_xcb_util_wm_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67"
+uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361"
+version = "0.4.1+1"
+
+[[Xorg_xkbcomp_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxkbfile_jll"]
+git-tree-sha1 = "4bcbf660f6c2e714f87e960a171b119d06ee163b"
+uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4"
+version = "1.4.2+4"
+
+[[Xorg_xkeyboard_config_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xkbcomp_jll"]
+git-tree-sha1 = "5c8424f8a67c3f2209646d4425f3d415fee5931d"
+uuid = "33bec58e-1273-512f-9401-5d533626f822"
+version = "2.27.0+4"
+
+[[Xorg_xtrans_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845"
+uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10"
+version = "1.4.0+3"
+
+[[ZipFile]]
+deps = ["Libdl", "Printf", "Zlib_jll"]
+git-tree-sha1 = "c3a5637e27e914a7a445b8d0ad063d701931e9f7"
+uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
+version = "0.9.3"
+
+[[Zlib_jll]]
+deps = ["Libdl"]
+uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+
+[[Zstd_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6"
+uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
+version = "1.5.0+0"
+
+[[Zygote]]
+deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"]
+git-tree-sha1 = "531474afbc343c3c7cb9b71c2771813c6defd550"
+uuid = "e88e6eb3-aa80-5325-afca-941959d7151f"
+version = "0.6.14"
+
+[[ZygoteRules]]
+deps = ["MacroTools"]
+git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7"
+uuid = "700de1a5-db45-46bc-99cf-38207098b444"
+version = "0.2.1"
+
+[[libass_jll]]
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "acc685bcf777b2202a904cdcb49ad34c2fa1880c"
+uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
+version = "0.14.0+4"
+
+[[libfdk_aac_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "7a5780a0d9c6864184b3a2eeeb833a0c871f00ab"
+uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280"
+version = "0.1.6+4"
+
+[[libpng_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c"
+uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f"
+version = "1.6.38+0"
+
+[[libvorbis_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"]
+git-tree-sha1 = "c45f4e40e7aafe9d086379e5578947ec8b95a8fb"
+uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a"
+version = "1.3.7+0"
+
+[[nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+
+[[p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+
+[[x264_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "d713c1ce4deac133e3334ee12f4adff07f81778f"
+uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a"
+version = "2020.7.14+2"
+
+[[x265_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "487da2f8f2f0c8ee0e83f39d13037d6bbf0a45ab"
+uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76"
+version = "3.0.0+3"
+
+[[xkbcommon_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"]
+git-tree-sha1 = "ece2350174195bb31de1a63bea3a41ae1aa593b6"
+uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd"
+version = "0.9.1+5"
diff --git a/examples/train-kernel-parameters/Project.toml b/examples/train-kernel-parameters/Project.toml
new file mode 100644
index 000000000..0645010d7
--- /dev/null
+++ b/examples/train-kernel-parameters/Project.toml
@@ -0,0 +1,17 @@
+[deps]
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+
+[compat]
+Distributions = "0.25"
+Flux = "0.12"
+ForwardDiff = "0.10"
+KernelFunctions = "0.10"
+Literate = "2"
+Plots = "1"
+julia = "1.3"
diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
new file mode 100644
index 000000000..c3cc5d1dd
--- /dev/null
+++ b/examples/train-kernel-parameters/script.jl
@@ -0,0 +1,84 @@
+# # Kernel Ridge Regression
+
+# ## We load KernelFunctions and some other packages
+
+using KernelFunctions
+using LinearAlgebra
+using Distributions
+using Plots;
+default(; lw=2.0, legendfontsize=15.0);
+using Flux: Optimise
+using ForwardDiff
+using Random: seed!
+seed!(42)
+
+# ## Data Generation
+# We generated data in 1 dimension
+
+xmin = -3;
+xmax = 3; # Bounds of the data
+N = 50 # Number of samples
+x_train = rand(Uniform(xmin, xmax), N) # We sample 100 random samples
+σ = 0.1
+y_train = sinc.(x_train) + randn(N) * σ # We create a function and add some noise
+x_test = range(xmin - 0.1, xmax + 0.1; length=300)
+
+# Plot the data
+
+scatter(x_train, y_train; lab="data")
+plot!(x_test, sinc; lab="true function")
+
+# ## Kernel training
+# To train the kernel parameters via ForwardDiff.jl
+# we need to create a function creating a kernel from an array
+
+kernelcall(θ) = transform(
+    exp(θ[1]) * SqExponentialKernel(),# + exp(θ[2]) * Matern32Kernel(),
+    exp(θ[3]),
+)
+
+# From theory we know the prediction for a test set x given
+# the kernel parameters and normalization constant
+
+function f(x, x_train, y_train, θ)
+    k = kernelcall(θ[1:3])
+    return kernelmatrix(k, x, x_train) *
+           ((kernelmatrix(k, x_train) + exp(θ[4]) * I) \ y_train)
+end
+
+# We look how the prediction looks like
+# with starting parameters [1.0, 1.0, 1.0, 1.0] we get :
+
+ŷ = f(x_test, x_train, y_train, log.(ones(4)))
+scatter(x_train, y_train; lab="data")
+plot!(x_test, sinc; lab="true function")
+plot!(x_test, ŷ; lab="prediction")
+
+# We define the loss based on the L2 norm both
+# for the loss and the regularization
+
+function loss(θ)
+    ŷ = f(x_train, x_train, y_train, θ)
+    return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
+end
+
+# The loss with our starting point :
+
+loss(log.(ones(4)))
+
+# ## Training the model
+
+θ = log.([1.0, 0.0, 0.01, 0.001]) # Initial vector
+anim = Animation()
+opt = Optimise.ADAGrad(0.5)
+for i in 1:30
+    grads = ForwardDiff.gradient(loss, θ) # We compute the gradients given the kernel parameters and regularization
+    Optimise.update!(opt, θ, grads)
+    scatter(
+        x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
+    )
+    plot!(x_test, sinc; lab="true function")
+    plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
+    frame(anim)
+end
+gif(anim)

From 2e54a35878236a363d393ab93272ac533cdb52d5 Mon Sep 17 00:00:00 2001
From: ST John <st--@users.noreply.github.com>
Date: Thu, 1 Jul 2021 14:12:49 +0300
Subject: [PATCH 02/41] update script

---
 examples/train-kernel-parameters/script.jl | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index c3cc5d1dd..6e591df61 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -32,10 +32,7 @@ plot!(x_test, sinc; lab="true function")
 # To train the kernel parameters via ForwardDiff.jl
 # we need to create a function creating a kernel from an array
 
-kernelcall(θ) = transform(
-    exp(θ[1]) * SqExponentialKernel(),# + exp(θ[2]) * Matern32Kernel(),
-    exp(θ[3]),
-)
+kernelcall(θ) = (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘ ScaleTransform(exp(θ[3]))
 
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant

From 74315f8ddc207d58d536a3f4a53a5478f909bee4 Mon Sep 17 00:00:00 2001
From: st-- <st--@users.noreply.github.com>
Date: Thu, 1 Jul 2021 15:55:05 +0300
Subject: [PATCH 03/41] Apply suggestions from code review

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 examples/train-kernel-parameters/script.jl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 6e591df61..65cfc3dfa 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -32,7 +32,10 @@ plot!(x_test, sinc; lab="true function")
 # To train the kernel parameters via ForwardDiff.jl
 # we need to create a function creating a kernel from an array
 
-kernelcall(θ) = (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘ ScaleTransform(exp(θ[3]))
+function kernelcall(θ)
+    return (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘
+           ScaleTransform(exp(θ[3]))
+end
 
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant

From 88f1b4d240990d1a0ab7784a5d621661f5459345 Mon Sep 17 00:00:00 2001
From: ST John <st--@users.noreply.github.com>
Date: Fri, 2 Jul 2021 11:19:03 +0300
Subject: [PATCH 04/41] fix out-of-domain initial value

---
 examples/train-kernel-parameters/script.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 65cfc3dfa..8966ac4cf 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -68,7 +68,7 @@ loss(log.(ones(4)))
 
 # ## Training the model
 
-θ = log.([1.0, 0.0, 0.01, 0.001]) # Initial vector
+θ = log.([1.0, 0.1, 0.01, 0.001]) # Initial vector
 anim = Animation()
 opt = Optimise.ADAGrad(0.5)
 for i in 1:30

From 12ee8accb1ffd85a2463d2b8587094780bc4d9c5 Mon Sep 17 00:00:00 2001
From: ST John <st--@users.noreply.github.com>
Date: Wed, 30 Jun 2021 17:13:02 +0300
Subject: [PATCH 05/41] initial version of training kernel parameters example
 from st/examples (#234)

---
 .../train-kernel-parameters/Manifest.toml     | 1088 +++++++++++++++++
 examples/train-kernel-parameters/Project.toml |   17 +
 examples/train-kernel-parameters/script.jl    |   84 ++
 3 files changed, 1189 insertions(+)
 create mode 100644 examples/train-kernel-parameters/Manifest.toml
 create mode 100644 examples/train-kernel-parameters/Project.toml
 create mode 100644 examples/train-kernel-parameters/script.jl

diff --git a/examples/train-kernel-parameters/Manifest.toml b/examples/train-kernel-parameters/Manifest.toml
new file mode 100644
index 000000000..7a3bb3837
--- /dev/null
+++ b/examples/train-kernel-parameters/Manifest.toml
@@ -0,0 +1,1088 @@
+# This file is machine-generated - editing it directly is not advised
+
+[[AbstractFFTs]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0"
+uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
+version = "1.0.1"
+
+[[AbstractTrees]]
+git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5"
+uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
+version = "0.3.4"
+
+[[Adapt]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7"
+uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
+version = "3.3.1"
+
+[[ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+
+[[Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
+[[BFloat16s]]
+deps = ["LinearAlgebra", "Test"]
+git-tree-sha1 = "4af69e205efc343068dc8722b8dfec1ade89254a"
+uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b"
+version = "0.1.0"
+
+[[Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[Bzip2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "c3598e525718abcc440f69cc6d5f60dda0a1b61e"
+uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
+version = "1.0.6+5"
+
+[[CEnum]]
+git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
+uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
+version = "0.4.1"
+
+[[CUDA]]
+deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"]
+git-tree-sha1 = "82b2811f5888465d96b38c7bb12d8fb9c25838e1"
+uuid = "052768ef-5323-5732-b1bb-66c8b64840ba"
+version = "3.3.1"
+
+[[Cairo_jll]]
+deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
+git-tree-sha1 = "e2f47f6d8337369411569fd45ae5753ca10394c6"
+uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
+version = "1.16.0+6"
+
+[[ChainRules]]
+deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Statistics"]
+git-tree-sha1 = "e7e3200bd24b77bcc849e6616f7c2f0d45d70f5b"
+uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2"
+version = "0.8.17"
+
+[[ChainRulesCore]]
+deps = ["Compat", "LinearAlgebra", "SparseArrays"]
+git-tree-sha1 = "be770c08881f7bb928dfd86d1ba83798f76cf62a"
+uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+version = "0.10.9"
+
+[[CodecZlib]]
+deps = ["TranscodingStreams", "Zlib_jll"]
+git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
+uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
+version = "0.7.0"
+
+[[ColorSchemes]]
+deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random", "StaticArrays"]
+git-tree-sha1 = "c8fd01e4b736013bc61b704871d20503b33ea402"
+uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
+version = "3.12.1"
+
+[[ColorTypes]]
+deps = ["FixedPointNumbers", "Random"]
+git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597"
+uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
+version = "0.11.0"
+
+[[Colors]]
+deps = ["ColorTypes", "FixedPointNumbers", "Reexport"]
+git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40"
+uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
+version = "0.12.8"
+
+[[CommonSubexpressions]]
+deps = ["MacroTools", "Test"]
+git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7"
+uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
+version = "0.3.0"
+
+[[Compat]]
+deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
+git-tree-sha1 = "dc7dedc2c2aa9faf59a55c622760a25cbefbe941"
+uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
+version = "3.31.0"
+
+[[CompilerSupportLibraries_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
+
+[[CompositionsBase]]
+git-tree-sha1 = "f3955eb38944e5dd0fabf8ca1e267d94941d34a5"
+uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
+version = "0.1.0"
+
+[[Contour]]
+deps = ["StaticArrays"]
+git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7"
+uuid = "d38c429a-6771-53c6-b99e-75d170b6e991"
+version = "0.5.7"
+
+[[DataAPI]]
+git-tree-sha1 = "ee400abb2298bd13bfc3df1c412ed228061a2385"
+uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+version = "1.7.0"
+
+[[DataStructures]]
+deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
+git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677"
+uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+version = "0.18.9"
+
+[[DataValueInterfaces]]
+git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
+uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
+version = "1.0.0"
+
+[[Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[DelimitedFiles]]
+deps = ["Mmap"]
+uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+
+[[DiffResults]]
+deps = ["StaticArrays"]
+git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805"
+uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
+version = "1.0.3"
+
+[[DiffRules]]
+deps = ["NaNMath", "Random", "SpecialFunctions"]
+git-tree-sha1 = "214c3fcac57755cfda163d91c58893a8723f93e9"
+uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
+version = "1.0.2"
+
+[[Distances]]
+deps = ["LinearAlgebra", "Statistics", "StatsAPI"]
+git-tree-sha1 = "abe4ad222b26af3337262b8afb28fab8d215e9f8"
+uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
+version = "0.10.3"
+
+[[Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+
+[[Distributions]]
+deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"]
+git-tree-sha1 = "2733323e5c02a9d7f48e7a3c4bc98d764fb704da"
+uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
+version = "0.25.6"
+
+[[DocStringExtensions]]
+deps = ["LibGit2"]
+git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
+uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+version = "0.8.5"
+
+[[Downloads]]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+
+[[EarCut_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "92d8f9f208637e8d2d28c664051a00569c01493d"
+uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5"
+version = "2.1.5+1"
+
+[[Expat_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "b3bfd02e98aedfa5cf885665493c5598c350cd2f"
+uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
+version = "2.2.10+0"
+
+[[ExprTools]]
+git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e"
+uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
+version = "0.1.3"
+
+[[FFMPEG]]
+deps = ["FFMPEG_jll"]
+git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8"
+uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a"
+version = "0.4.1"
+
+[[FFMPEG_jll]]
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "LibVPX_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
+git-tree-sha1 = "3cc57ad0a213808473eafef4845a74766242e05f"
+uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
+version = "4.3.1+4"
+
+[[FillArrays]]
+deps = ["LinearAlgebra", "Random", "SparseArrays"]
+git-tree-sha1 = "a603e79b71bb3c1efdb58f0ee32286efe2d1a255"
+uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
+version = "0.11.8"
+
+[[FixedPointNumbers]]
+deps = ["Statistics"]
+git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc"
+uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
+version = "0.8.4"
+
+[[Flux]]
+deps = ["AbstractTrees", "Adapt", "CUDA", "CodecZlib", "Colors", "DelimitedFiles", "Functors", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "NNlibCUDA", "Pkg", "Printf", "Random", "Reexport", "SHA", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"]
+git-tree-sha1 = "0b3c6d0ce57d3b793eabd346ccc8f605035ef079"
+uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+version = "0.12.4"
+
+[[Fontconfig_jll]]
+deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "35895cf184ceaab11fd778b4590144034a167a2f"
+uuid = "a3f928ae-7b40-5064-980b-68af3947d34b"
+version = "2.13.1+14"
+
+[[Formatting]]
+deps = ["Printf"]
+git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
+uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
+version = "0.4.2"
+
+[[ForwardDiff]]
+deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
+git-tree-sha1 = "e2af66012e08966366a43251e1fd421522908be6"
+uuid = "f6369f11-7733-5829-9624-2563aa707210"
+version = "0.10.18"
+
+[[FreeType2_jll]]
+deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "cbd58c9deb1d304f5a245a0b7eb841a2560cfec6"
+uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7"
+version = "2.10.1+5"
+
+[[FriBidi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91"
+uuid = "559328eb-81f9-559d-9380-de523a88c83c"
+version = "1.0.10+0"
+
+[[Functors]]
+deps = ["MacroTools"]
+git-tree-sha1 = "a7bb2af991c43dcf5c3455d276dd83976799634f"
+uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
+version = "0.2.1"
+
+[[GLFW_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"]
+git-tree-sha1 = "dba1e8614e98949abfa60480b13653813d8f0157"
+uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89"
+version = "3.3.5+0"
+
+[[GPUArrays]]
+deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"]
+git-tree-sha1 = "ececbf05f8904c92814bdbd0aafd5540b0bf2e9a"
+uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
+version = "7.0.1"
+
+[[GPUCompiler]]
+deps = ["DataStructures", "ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "TimerOutputs", "UUIDs"]
+git-tree-sha1 = "222c6cdb888ec24795936d6829aa978691def60e"
+uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
+version = "0.12.3"
+
+[[GR]]
+deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"]
+git-tree-sha1 = "b83e3125048a9c3158cbb7ca423790c7b1b57bea"
+uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
+version = "0.57.5"
+
+[[GR_jll]]
+deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"]
+git-tree-sha1 = "e14907859a1d3aee73a019e7b3c98e9e7b8b5b3e"
+uuid = "d2c73de3-f751-5644-a686-071e5b155ba9"
+version = "0.57.3+0"
+
+[[GeometryBasics]]
+deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"]
+git-tree-sha1 = "15ff9a14b9e1218958d3530cc288cf31465d9ae2"
+uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
+version = "0.3.13"
+
+[[Gettext_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"]
+git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046"
+uuid = "78b55507-aeef-58d4-861c-77aaff3498b1"
+version = "0.21.0+0"
+
+[[Glib_jll]]
+deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "47ce50b742921377301e15005c96e979574e130b"
+uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
+version = "2.68.1+0"
+
+[[Grisu]]
+git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2"
+uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe"
+version = "1.0.2"
+
+[[HTTP]]
+deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
+git-tree-sha1 = "99c6bc381e98c5331f34b9ce949aebea209bd0db"
+uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+version = "0.9.11"
+
+[[IOCapture]]
+deps = ["Logging", "Random"]
+git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
+uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
+version = "0.2.2"
+
+[[IRTools]]
+deps = ["InteractiveUtils", "MacroTools", "Test"]
+git-tree-sha1 = "95215cd0076a150ef46ff7928892bc341864c73c"
+uuid = "7869d1d1-7146-5819-86e3-90919afe41df"
+version = "0.4.3"
+
+[[IniFile]]
+deps = ["Test"]
+git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
+uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
+version = "0.5.0"
+
+[[InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[IterTools]]
+git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18"
+uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
+version = "1.3.0"
+
+[[IteratorInterfaceExtensions]]
+git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
+uuid = "82899510-4779-5014-852e-03e436cf321d"
+version = "1.0.0"
+
+[[JLLWrappers]]
+deps = ["Preferences"]
+git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
+uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
+version = "1.3.0"
+
+[[JSON]]
+deps = ["Dates", "Mmap", "Parsers", "Unicode"]
+git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.21.1"
+
+[[JpegTurbo_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "d735490ac75c5cb9f1b00d8b5509c11984dc6943"
+uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8"
+version = "2.1.0+0"
+
+[[Juno]]
+deps = ["Base64", "Logging", "Media", "Profile"]
+git-tree-sha1 = "07cb43290a840908a771552911a6274bc6c072c7"
+uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d"
+version = "0.8.4"
+
+[[KernelFunctions]]
+deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "LinearAlgebra", "Random", "Requires", "SpecialFunctions", "StatsBase", "StatsFuns", "TensorCore", "Test", "ZygoteRules"]
+git-tree-sha1 = "e8b5ba31b6d18695fd46bfcd8557682839023195"
+uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
+version = "0.10.6"
+
+[[LAME_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c"
+uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d"
+version = "3.100.1+0"
+
+[[LLVM]]
+deps = ["CEnum", "Libdl", "Printf", "Unicode"]
+git-tree-sha1 = "f57ac3fd2045b50d3db081663837ac5b4096947e"
+uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
+version = "3.9.0"
+
+[[LZO_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6"
+uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
+version = "2.10.1+0"
+
+[[LaTeXStrings]]
+git-tree-sha1 = "c7f1c695e06c01b95a67f0cd1d34994f3e7db104"
+uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
+version = "1.2.1"
+
+[[Latexify]]
+deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"]
+git-tree-sha1 = "a4b12a1bd2ebade87891ab7e36fdbce582301a92"
+uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
+version = "0.15.6"
+
+[[LazyArtifacts]]
+deps = ["Artifacts", "Pkg"]
+uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
+
+[[LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+
+[[LibCURL_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+
+[[LibGit2]]
+deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+
+[[LibVPX_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "12ee7e23fa4d18361e7c2cde8f8337d4c3101bc7"
+uuid = "dd192d2f-8180-539f-9fb4-cc70b1dcf69a"
+version = "1.10.0+0"
+
+[[Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[Libffi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "761a393aeccd6aa92ec3515e428c26bf99575b3b"
+uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490"
+version = "3.2.2+0"
+
+[[Libgcrypt_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"]
+git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae"
+uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4"
+version = "1.8.7+0"
+
+[[Libglvnd_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_libXext_jll"]
+git-tree-sha1 = "7739f837d6447403596a75d19ed01fd08d6f56bf"
+uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29"
+version = "1.3.0+3"
+
+[[Libgpg_error_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9"
+uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8"
+version = "1.42.0+0"
+
+[[Libiconv_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778"
+uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
+version = "1.16.1+1"
+
+[[Libmount_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73"
+uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9"
+version = "2.35.0+0"
+
+[[Libtiff_jll]]
+deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"]
+git-tree-sha1 = "340e257aada13f95f98ee352d316c3bed37c8ab9"
+uuid = "89763e89-9b03-5906-acba-b20f662cd828"
+version = "4.3.0+0"
+
+[[Libuuid_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066"
+uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700"
+version = "2.36.0+0"
+
+[[LinearAlgebra]]
+deps = ["Libdl"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
+[[Literate]]
+deps = ["Base64", "IOCapture", "JSON", "REPL"]
+git-tree-sha1 = "501a1a74a0c825037860d36d87d703e987d39dbc"
+uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+version = "2.8.1"
+
+[[LogExpFunctions]]
+deps = ["DocStringExtensions", "LinearAlgebra"]
+git-tree-sha1 = "1ba664552f1ef15325e68dc4c05c3ef8c2d5d885"
+uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
+version = "0.2.4"
+
+[[Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[MacroTools]]
+deps = ["Markdown", "Random"]
+git-tree-sha1 = "6a8a2a625ab0dea913aba95c11370589e0239ff0"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.6"
+
+[[Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[MbedTLS]]
+deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"]
+git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe"
+uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
+version = "1.0.3"
+
+[[MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+
+[[Measures]]
+git-tree-sha1 = "e498ddeee6f9fdb4551ce855a46f54dbd900245f"
+uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e"
+version = "0.3.1"
+
+[[Media]]
+deps = ["MacroTools", "Test"]
+git-tree-sha1 = "75a54abd10709c01f1b86b84ec225d26e840ed58"
+uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27"
+version = "0.5.0"
+
+[[Missings]]
+deps = ["DataAPI"]
+git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7"
+uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
+version = "1.0.0"
+
+[[Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"
+
+[[MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+
+[[NNlib]]
+deps = ["Adapt", "ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"]
+git-tree-sha1 = "7461639cef384a2ad058005b49e32b318d844343"
+uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+version = "0.7.22"
+
+[[NNlibCUDA]]
+deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"]
+git-tree-sha1 = "c2f2f152340022ce44c6ac97e05c62a6de59c373"
+uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d"
+version = "0.1.4"
+
+[[NaNMath]]
+git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb"
+uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
+version = "0.3.5"
+
+[[NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+
+[[Ogg_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f"
+uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051"
+version = "1.3.5+0"
+
+[[OpenSSL_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a"
+uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
+version = "1.1.10+0"
+
+[[OpenSpecFun_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
+uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
+version = "0.5.5+0"
+
+[[Opus_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720"
+uuid = "91d4177d-7536-5919-b921-800302f37372"
+version = "1.3.2+0"
+
+[[OrderedCollections]]
+git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.4.1"
+
+[[PCRE_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "b2a7af664e098055a7529ad1a900ded962bca488"
+uuid = "2f80f16e-611a-54ab-bc61-aa92de5b98fc"
+version = "8.44.0+0"
+
+[[PDMats]]
+deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
+git-tree-sha1 = "4dd403333bcf0909341cfe57ec115152f937d7d8"
+uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
+version = "0.11.1"
+
+[[Parsers]]
+deps = ["Dates"]
+git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc"
+uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+version = "1.1.0"
+
+[[Pixman_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "b4f5d02549a10e20780a24fce72bea96b6329e29"
+uuid = "30392449-352a-5448-841d-b1acce4e97dc"
+version = "0.40.1+0"
+
+[[Pkg]]
+deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+
+[[PlotThemes]]
+deps = ["PlotUtils", "Requires", "Statistics"]
+git-tree-sha1 = "a3a964ce9dc7898193536002a6dd892b1b5a6f1d"
+uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a"
+version = "2.0.1"
+
+[[PlotUtils]]
+deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"]
+git-tree-sha1 = "ae9a295ac761f64d8c2ec7f9f24d21eb4ffba34d"
+uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043"
+version = "1.0.10"
+
+[[Plots]]
+deps = ["Base64", "Contour", "Dates", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"]
+git-tree-sha1 = "df601eed7c9637235a26b26f9f648deccd277178"
+uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+version = "1.16.7"
+
+[[Preferences]]
+deps = ["TOML"]
+git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+version = "1.2.2"
+
+[[Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[Profile]]
+deps = ["Printf"]
+uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
+
+[[Qt5Base_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"]
+git-tree-sha1 = "ad368663a5e20dbb8d6dc2fddeefe4dae0781ae8"
+uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1"
+version = "5.15.3+0"
+
+[[QuadGK]]
+deps = ["DataStructures", "LinearAlgebra"]
+git-tree-sha1 = "12fbe86da16df6679be7521dfb39fbc861e1dc7b"
+uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
+version = "2.4.1"
+
+[[REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[Random]]
+deps = ["Serialization"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[Random123]]
+deps = ["Libdl", "Random", "RandomNumbers"]
+git-tree-sha1 = "0e8b146557ad1c6deb1367655e052276690e71a3"
+uuid = "74087812-796a-5b5d-8853-05524746bad3"
+version = "1.4.2"
+
+[[RandomNumbers]]
+deps = ["Random", "Requires"]
+git-tree-sha1 = "441e6fc35597524ada7f85e13df1f4e10137d16f"
+uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143"
+version = "1.4.0"
+
+[[RecipesBase]]
+git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae"
+uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
+version = "1.1.1"
+
+[[RecipesPipeline]]
+deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"]
+git-tree-sha1 = "9b8e57e3cca8828a1bc759840bfe48d64db9abfb"
+uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c"
+version = "0.3.3"
+
+[[Reexport]]
+git-tree-sha1 = "5f6c21241f0f655da3952fd60aa18477cf96c220"
+uuid = "189a3867-3050-52da-a836-e630ba90ab69"
+version = "1.1.0"
+
+[[Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.1.3"
+
+[[Rmath]]
+deps = ["Random", "Rmath_jll"]
+git-tree-sha1 = "bf3188feca147ce108c76ad82c2792c57abe7b1f"
+uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
+version = "0.7.0"
+
+[[Rmath_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "68db32dff12bb6127bac73c209881191bf0efbb7"
+uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
+version = "0.3.0+0"
+
+[[SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+
+[[Scratch]]
+deps = ["Dates"]
+git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda"
+uuid = "6c6a2e73-6563-6170-7368-637461726353"
+version = "1.1.0"
+
+[[Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[SharedArrays]]
+deps = ["Distributed", "Mmap", "Random", "Serialization"]
+uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
+
+[[Showoff]]
+deps = ["Dates", "Grisu"]
+git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de"
+uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f"
+version = "1.0.3"
+
+[[Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[SortingAlgorithms]]
+deps = ["DataStructures"]
+git-tree-sha1 = "2ec1962eba973f383239da22e75218565c390a96"
+uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
+version = "1.0.0"
+
+[[SparseArrays]]
+deps = ["LinearAlgebra", "Random"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+
+[[SpecialFunctions]]
+deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"]
+git-tree-sha1 = "a50550fa3164a8c46747e62063b4d774ac1bcf49"
+uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
+version = "1.5.1"
+
+[[StaticArrays]]
+deps = ["LinearAlgebra", "Random", "Statistics"]
+git-tree-sha1 = "745914ebcd610da69f3cb6bf76cb7bb83dcb8c9a"
+uuid = "90137ffa-7385-5640-81b9-e52037218182"
+version = "1.2.4"
+
+[[Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[[StatsAPI]]
+git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
+uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
+version = "1.0.0"
+
+[[StatsBase]]
+deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "2f6792d523d7448bbe2fec99eca9218f06cc746d"
+uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+version = "0.33.8"
+
+[[StatsFuns]]
+deps = ["LogExpFunctions", "Rmath", "SpecialFunctions"]
+git-tree-sha1 = "30cd8c360c54081f806b1ee14d2eecbef3c04c49"
+uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
+version = "0.9.8"
+
+[[StructArrays]]
+deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"]
+git-tree-sha1 = "000e168f5cc9aded17b6999a560b7c11dda69095"
+uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
+version = "0.6.0"
+
+[[SuiteSparse]]
+deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
+uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
+
+[[TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+
+[[TableTraits]]
+deps = ["IteratorInterfaceExtensions"]
+git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
+uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
+version = "1.0.1"
+
+[[Tables]]
+deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
+git-tree-sha1 = "8ed4a3ea724dac32670b062be3ef1c1de6773ae8"
+uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+version = "1.4.4"
+
+[[Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+
+[[TensorCore]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6"
+uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50"
+version = "0.1.1"
+
+[[Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[TimerOutputs]]
+deps = ["ExprTools", "Printf"]
+git-tree-sha1 = "9f494bc54b4c31404a9eff449235836615929de1"
+uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
+version = "0.5.10"
+
+[[TranscodingStreams]]
+deps = ["Random", "Test"]
+git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
+uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
+version = "0.9.5"
+
+[[URIs]]
+git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355"
+uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
+version = "1.3.0"
+
+[[UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[Wayland_jll]]
+deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"]
+git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23"
+uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89"
+version = "1.19.0+0"
+
+[[Wayland_protocols_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll"]
+git-tree-sha1 = "2839f1c1296940218e35df0bbb220f2a79686670"
+uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91"
+version = "1.18.0+4"
+
+[[XML2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a"
+uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
+version = "2.9.12+0"
+
+[[XSLT_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"]
+git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a"
+uuid = "aed1982a-8fda-507f-9586-7b0439959a61"
+version = "1.1.34+0"
+
+[[Xorg_libX11_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll", "Xorg_xtrans_jll"]
+git-tree-sha1 = "5be649d550f3f4b95308bf0183b82e2582876527"
+uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc"
+version = "1.6.9+4"
+
+[[Xorg_libXau_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "4e490d5c960c314f33885790ed410ff3a94ce67e"
+uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec"
+version = "1.0.9+4"
+
+[[Xorg_libXcursor_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"]
+git-tree-sha1 = "12e0eb3bc634fa2080c1c37fccf56f7c22989afd"
+uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724"
+version = "1.2.0+4"
+
+[[Xorg_libXdmcp_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "4fe47bd2247248125c428978740e18a681372dd4"
+uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05"
+version = "1.1.3+4"
+
+[[Xorg_libXext_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3"
+uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3"
+version = "1.3.4+4"
+
+[[Xorg_libXfixes_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "0e0dc7431e7a0587559f9294aeec269471c991a4"
+uuid = "d091e8ba-531a-589c-9de9-94069b037ed8"
+version = "5.0.3+4"
+
+[[Xorg_libXi_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll"]
+git-tree-sha1 = "89b52bc2160aadc84d707093930ef0bffa641246"
+uuid = "a51aa0fd-4e3c-5386-b890-e753decda492"
+version = "1.7.10+4"
+
+[[Xorg_libXinerama_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll"]
+git-tree-sha1 = "26be8b1c342929259317d8b9f7b53bf2bb73b123"
+uuid = "d1454406-59df-5ea1-beac-c340f2130bc3"
+version = "1.1.4+4"
+
+[[Xorg_libXrandr_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll"]
+git-tree-sha1 = "34cea83cb726fb58f325887bf0612c6b3fb17631"
+uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484"
+version = "1.5.2+4"
+
+[[Xorg_libXrender_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96"
+uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa"
+version = "0.9.10+4"
+
+[[Xorg_libpthread_stubs_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "6783737e45d3c59a4a4c4091f5f88cdcf0908cbb"
+uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74"
+version = "0.1.0+3"
+
+[[Xorg_libxcb_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"]
+git-tree-sha1 = "daf17f441228e7a3833846cd048892861cff16d6"
+uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b"
+version = "1.13.0+3"
+
+[[Xorg_libxkbfile_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
+git-tree-sha1 = "926af861744212db0eb001d9e40b5d16292080b2"
+uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a"
+version = "1.1.0+4"
+
+[[Xorg_xcb_util_image_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97"
+uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b"
+version = "0.4.0+1"
+
+[[Xorg_xcb_util_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"]
+git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1"
+uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5"
+version = "0.4.0+1"
+
+[[Xorg_xcb_util_keysyms_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00"
+uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7"
+version = "0.4.0+1"
+
+[[Xorg_xcb_util_renderutil_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e"
+uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e"
+version = "0.3.9+1"
+
+[[Xorg_xcb_util_wm_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
+git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67"
+uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361"
+version = "0.4.1+1"
+
+[[Xorg_xkbcomp_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxkbfile_jll"]
+git-tree-sha1 = "4bcbf660f6c2e714f87e960a171b119d06ee163b"
+uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4"
+version = "1.4.2+4"
+
+[[Xorg_xkeyboard_config_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xkbcomp_jll"]
+git-tree-sha1 = "5c8424f8a67c3f2209646d4425f3d415fee5931d"
+uuid = "33bec58e-1273-512f-9401-5d533626f822"
+version = "2.27.0+4"
+
+[[Xorg_xtrans_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845"
+uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10"
+version = "1.4.0+3"
+
+[[ZipFile]]
+deps = ["Libdl", "Printf", "Zlib_jll"]
+git-tree-sha1 = "c3a5637e27e914a7a445b8d0ad063d701931e9f7"
+uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
+version = "0.9.3"
+
+[[Zlib_jll]]
+deps = ["Libdl"]
+uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+
+[[Zstd_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6"
+uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
+version = "1.5.0+0"
+
+[[Zygote]]
+deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"]
+git-tree-sha1 = "531474afbc343c3c7cb9b71c2771813c6defd550"
+uuid = "e88e6eb3-aa80-5325-afca-941959d7151f"
+version = "0.6.14"
+
+[[ZygoteRules]]
+deps = ["MacroTools"]
+git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7"
+uuid = "700de1a5-db45-46bc-99cf-38207098b444"
+version = "0.2.1"
+
+[[libass_jll]]
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "acc685bcf777b2202a904cdcb49ad34c2fa1880c"
+uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
+version = "0.14.0+4"
+
+[[libfdk_aac_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "7a5780a0d9c6864184b3a2eeeb833a0c871f00ab"
+uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280"
+version = "0.1.6+4"
+
+[[libpng_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c"
+uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f"
+version = "1.6.38+0"
+
+[[libvorbis_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"]
+git-tree-sha1 = "c45f4e40e7aafe9d086379e5578947ec8b95a8fb"
+uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a"
+version = "1.3.7+0"
+
+[[nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+
+[[p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+
+[[x264_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "d713c1ce4deac133e3334ee12f4adff07f81778f"
+uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a"
+version = "2020.7.14+2"
+
+[[x265_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "487da2f8f2f0c8ee0e83f39d13037d6bbf0a45ab"
+uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76"
+version = "3.0.0+3"
+
+[[xkbcommon_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"]
+git-tree-sha1 = "ece2350174195bb31de1a63bea3a41ae1aa593b6"
+uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd"
+version = "0.9.1+5"
diff --git a/examples/train-kernel-parameters/Project.toml b/examples/train-kernel-parameters/Project.toml
new file mode 100644
index 000000000..0645010d7
--- /dev/null
+++ b/examples/train-kernel-parameters/Project.toml
@@ -0,0 +1,17 @@
+[deps]
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+
+[compat]
+Distributions = "0.25"
+Flux = "0.12"
+ForwardDiff = "0.10"
+KernelFunctions = "0.10"
+Literate = "2"
+Plots = "1"
+julia = "1.3"
diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
new file mode 100644
index 000000000..c3cc5d1dd
--- /dev/null
+++ b/examples/train-kernel-parameters/script.jl
@@ -0,0 +1,84 @@
+# # Kernel Ridge Regression
+
+# ## We load KernelFunctions and some other packages
+
+using KernelFunctions
+using LinearAlgebra
+using Distributions
+using Plots;
+default(; lw=2.0, legendfontsize=15.0);
+using Flux: Optimise
+using ForwardDiff
+using Random: seed!
+seed!(42)
+
+# ## Data Generation
+# We generated data in 1 dimension
+
+xmin = -3;
+xmax = 3; # Bounds of the data
+N = 50 # Number of samples
+x_train = rand(Uniform(xmin, xmax), N) # We sample 100 random samples
+σ = 0.1
+y_train = sinc.(x_train) + randn(N) * σ # We create a function and add some noise
+x_test = range(xmin - 0.1, xmax + 0.1; length=300)
+
+# Plot the data
+
+scatter(x_train, y_train; lab="data")
+plot!(x_test, sinc; lab="true function")
+
+# ## Kernel training
+# To train the kernel parameters via ForwardDiff.jl
+# we need to create a function creating a kernel from an array
+
+kernelcall(θ) = transform(
+    exp(θ[1]) * SqExponentialKernel(),# + exp(θ[2]) * Matern32Kernel(),
+    exp(θ[3]),
+)
+
+# From theory we know the prediction for a test set x given
+# the kernel parameters and normalization constant
+
+function f(x, x_train, y_train, θ)
+    k = kernelcall(θ[1:3])
+    return kernelmatrix(k, x, x_train) *
+           ((kernelmatrix(k, x_train) + exp(θ[4]) * I) \ y_train)
+end
+
+# We look how the prediction looks like
+# with starting parameters [1.0, 1.0, 1.0, 1.0] we get :
+
+ŷ = f(x_test, x_train, y_train, log.(ones(4)))
+scatter(x_train, y_train; lab="data")
+plot!(x_test, sinc; lab="true function")
+plot!(x_test, ŷ; lab="prediction")
+
+# We define the loss based on the L2 norm both
+# for the loss and the regularization
+
+function loss(θ)
+    ŷ = f(x_train, x_train, y_train, θ)
+    return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
+end
+
+# The loss with our starting point :
+
+loss(log.(ones(4)))
+
+# ## Training the model
+
+θ = log.([1.0, 0.0, 0.01, 0.001]) # Initial vector
+anim = Animation()
+opt = Optimise.ADAGrad(0.5)
+for i in 1:30
+    grads = ForwardDiff.gradient(loss, θ) # We compute the gradients given the kernel parameters and regularization
+    Optimise.update!(opt, θ, grads)
+    scatter(
+        x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
+    )
+    plot!(x_test, sinc; lab="true function")
+    plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
+    frame(anim)
+end
+gif(anim)

From 0c76b7a22eeb01c472a6f8754ea0b2c38a476fc7 Mon Sep 17 00:00:00 2001
From: ST John <st--@users.noreply.github.com>
Date: Thu, 1 Jul 2021 14:12:49 +0300
Subject: [PATCH 06/41] update script

---
 examples/train-kernel-parameters/script.jl | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index c3cc5d1dd..6e591df61 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -32,10 +32,7 @@ plot!(x_test, sinc; lab="true function")
 # To train the kernel parameters via ForwardDiff.jl
 # we need to create a function creating a kernel from an array
 
-kernelcall(θ) = transform(
-    exp(θ[1]) * SqExponentialKernel(),# + exp(θ[2]) * Matern32Kernel(),
-    exp(θ[3]),
-)
+kernelcall(θ) = (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘ ScaleTransform(exp(θ[3]))
 
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant

From 3569b5779b490686c688bb99de5b405d724739b8 Mon Sep 17 00:00:00 2001
From: st-- <st--@users.noreply.github.com>
Date: Thu, 1 Jul 2021 15:55:05 +0300
Subject: [PATCH 07/41] Apply suggestions from code review

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 examples/train-kernel-parameters/script.jl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 6e591df61..65cfc3dfa 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -32,7 +32,10 @@ plot!(x_test, sinc; lab="true function")
 # To train the kernel parameters via ForwardDiff.jl
 # we need to create a function creating a kernel from an array
 
-kernelcall(θ) = (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘ ScaleTransform(exp(θ[3]))
+function kernelcall(θ)
+    return (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘
+           ScaleTransform(exp(θ[3]))
+end
 
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant

From 9e1f96acbc60e99c841e36b94c78c06725e5c22a Mon Sep 17 00:00:00 2001
From: ST John <st--@users.noreply.github.com>
Date: Fri, 2 Jul 2021 11:19:03 +0300
Subject: [PATCH 08/41] fix out-of-domain initial value

---
 examples/train-kernel-parameters/script.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 65cfc3dfa..8966ac4cf 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -68,7 +68,7 @@ loss(log.(ones(4)))
 
 # ## Training the model
 
-θ = log.([1.0, 0.0, 0.01, 0.001]) # Initial vector
+θ = log.([1.0, 0.1, 0.01, 0.001]) # Initial vector
 anim = Animation()
 opt = Optimise.ADAGrad(0.5)
 for i in 1:30

From 30171c98357f236fddbd4a0999e382bd148d8af7 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Tue, 26 Oct 2021 02:14:13 +0100
Subject: [PATCH 09/41] Add ParameterHandling

---
 examples/train-kernel-parameters/script.jl | 74 ++++++++++++++++++++--
 1 file changed, 70 insertions(+), 4 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 8966ac4cf..4b8ff4ac0 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -1,5 +1,7 @@
 # # Kernel Ridge Regression
 
+# In this example we show the two main methods to perform regression on a kernel from KernelFunctions.jl.
+
 # ## We load KernelFunctions and some other packages
 
 using KernelFunctions
@@ -11,6 +13,7 @@ using Flux: Optimise
 using ForwardDiff
 using Random: seed!
 seed!(42)
+using ParameterHandling
 
 # ## Data Generation
 # We generated data in 1 dimension
@@ -28,9 +31,18 @@ x_test = range(xmin - 0.1, xmax + 0.1; length=300)
 scatter(x_train, y_train; lab="data")
 plot!(x_test, sinc; lab="true function")
 
-# ## Kernel training
+# ## Kernel training, Method 1
+# The first method is to rebuild the parametrized kernel from a vector of parameters 
+# in each evaluation of the cost fuction. This is similar to the approach taken in 
+# [Stheno.jl](https://github.com/JuliaGaussianProcesses/Stheno.jl).
+
+
+# ### Simplest Approach
+# A simple way to ensure that the kernel parameters are positive
+# is to optimize over the logarithm of the parameters. 
+
 # To train the kernel parameters via ForwardDiff.jl
-# we need to create a function creating a kernel from an array
+# we need to create a function creating a kernel from an array.
 
 function kernelcall(θ)
     return (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘
@@ -54,6 +66,7 @@ scatter(x_train, y_train; lab="data")
 plot!(x_test, sinc; lab="true function")
 plot!(x_test, ŷ; lab="prediction")
 
+
 # We define the loss based on the L2 norm both
 # for the loss and the regularization
 
@@ -68,11 +81,13 @@ loss(log.(ones(4)))
 
 # ## Training the model
 
-θ = log.([1.0, 0.1, 0.01, 0.001]) # Initial vector
+θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
+# θ = positive.([1.1, 0.1, 0.01, 0.001]) 
 anim = Animation()
 opt = Optimise.ADAGrad(0.5)
 for i in 1:30
-    grads = ForwardDiff.gradient(loss, θ) # We compute the gradients given the kernel parameters and regularization
+    println(i)
+    grads = only(Zygote.gradient(loss, θ)) # We compute the gradients given the kernel parameters and regularization
     Optimise.update!(opt, θ, grads)
     scatter(
         x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
@@ -82,3 +97,54 @@ for i in 1:30
     frame(anim)
 end
 gif(anim)
+
+# ### ParameterHandling.jl
+# Alternatively, we can use the [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl) package.
+
+raw_initial_θ = (
+    k1 = positive(1.1),
+    k2 = positive(0.1),
+    k3 = positive(0.01),
+    noise_var=positive(0.001),
+)
+
+flat_θ, unflatten = ParameterHandling.value_flatten(raw_initial_θ);
+
+function kernelcall(θ)
+    return (θ.k1 * SqExponentialKernel() + θ.k2 * Matern32Kernel()) ∘
+           ScaleTransform(θ.k3)
+end
+
+function f(x, x_train, y_train, θ)
+    k = kernelcall(θ)
+    return kernelmatrix(k, x, x_train) *
+           ((kernelmatrix(k, x_train) + θ.noise_var * I) \ y_train)
+end
+
+function loss(θ)
+    ŷ = f(x_train, x_train, y_train, θ)
+    return sum(abs2, y_train - ŷ) + θ.noise_var * norm(ŷ)
+end
+
+initial_θ = ParameterHandling.value(raw_initial_θ)
+
+# The loss with our starting point :
+
+loss(initial_θ)
+
+# ## Training the model
+
+anim = Animation()
+opt = Optimise.ADAGrad(0.5)
+for i in 1:30
+    println(i)
+    grads = only(Zygote.gradient(loss ∘ unflatten, flat_θ)) # We compute the gradients given the kernel parameters and regularization
+    Optimise.update!(opt, flat_θ, grads)
+    scatter(
+        x_train, y_train; lab="data", title="i = $(i), Loss = $(round((loss ∘ unflatten)(flat_θ), digits = 4))"
+    )
+    plot!(x_test, sinc; lab="true function")
+    plot!(x_test, f(x_test, x_train, y_train, unflatten(flat_θ)); lab="Prediction", lw=3.0)
+    frame(anim)
+end
+gif(anim)
\ No newline at end of file

From a6d6e5bd19582193ab979eb7ffdd522a67268493 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Mon, 1 Nov 2021 00:31:40 +0000
Subject: [PATCH 10/41] Extend example

---
 examples/train-kernel-parameters/script.jl | 31 +++++++++++++++++-----
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 4b8ff4ac0..7134f6de0 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -31,13 +31,13 @@ x_test = range(xmin - 0.1, xmax + 0.1; length=300)
 scatter(x_train, y_train; lab="data")
 plot!(x_test, sinc; lab="true function")
 
-# ## Kernel training, Method 1
+# ## Method 1
 # The first method is to rebuild the parametrized kernel from a vector of parameters 
 # in each evaluation of the cost fuction. This is similar to the approach taken in 
 # [Stheno.jl](https://github.com/JuliaGaussianProcesses/Stheno.jl).
 
 
-# ### Simplest Approach
+# ### Base Approach
 # A simple way to ensure that the kernel parameters are positive
 # is to optimize over the logarithm of the parameters. 
 
@@ -86,7 +86,6 @@ loss(log.(ones(4)))
 anim = Animation()
 opt = Optimise.ADAGrad(0.5)
 for i in 1:30
-    println(i)
     grads = only(Zygote.gradient(loss, θ)) # We compute the gradients given the kernel parameters and regularization
     Optimise.update!(opt, θ, grads)
     scatter(
@@ -99,7 +98,8 @@ end
 gif(anim)
 
 # ### ParameterHandling.jl
-# Alternatively, we can use the [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl) package.
+# Alternatively, we can use the [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl) package 
+# to handle the requirement that all kernel parameters should be positive. 
 
 raw_initial_θ = (
     k1 = positive(1.1),
@@ -137,7 +137,6 @@ loss(initial_θ)
 anim = Animation()
 opt = Optimise.ADAGrad(0.5)
 for i in 1:30
-    println(i)
     grads = only(Zygote.gradient(loss ∘ unflatten, flat_θ)) # We compute the gradients given the kernel parameters and regularization
     Optimise.update!(opt, flat_θ, grads)
     scatter(
@@ -147,4 +146,24 @@ for i in 1:30
     plot!(x_test, f(x_test, x_train, y_train, unflatten(flat_θ)); lab="Prediction", lw=3.0)
     frame(anim)
 end
-gif(anim)
\ No newline at end of file
+gif(anim)
+
+
+# ## Method 2: Functor
+# An alternative method is to use tools from Flux.jl, which is a fairly heavy package. 
+
+# raw_initial_θ = (
+#     k1 = positive(1.1),
+#     k2 = positive(0.1),
+#     k3 = positive(0.01),
+#     noise_var=positive(0.001),
+# )
+k1 = 1.1
+k2 = 0.1
+k3 = 0.01
+noise_var = 0.001
+
+kernel = (k1 * SqExponentialKernel() + k2 * Matern32Kernel()) ∘
+           ScaleTransform(k3)
+
+Θ = Flux.params(k1, k2, k3) 
\ No newline at end of file

From f512e47c605afa6b8cfa4b559c18df099d0631cb Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Thu, 27 Jan 2022 23:07:33 +0000
Subject: [PATCH 11/41] Failed attempts with default Flux opt

---
 examples/train-kernel-parameters/script.jl | 121 ++++++++++++++++-----
 1 file changed, 94 insertions(+), 27 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 7134f6de0..6db7cb81e 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -9,8 +9,10 @@ using LinearAlgebra
 using Distributions
 using Plots;
 default(; lw=2.0, legendfontsize=15.0);
+using BenchmarkTools
+using Flux
 using Flux: Optimise
-using ForwardDiff
+using Zygote
 using Random: seed!
 seed!(42)
 using ParameterHandling
@@ -75,18 +77,27 @@ function loss(θ)
     return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
 end
 
+# ## Training the model
+
+θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
+opt = Optimise.ADAGrad(0.5)
+
 # The loss with our starting point :
 
-loss(log.(ones(4)))
+loss(θ)
 
-# ## Training the model
+# Cost for one step
+
+@benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
+    grads = only((Zygote.gradient(loss, θt))) # We compute the gradients given the kernel parameters and regularization
+    Optimise.update!(optt, θt, grads)
+end
+
+# The optimization 
 
-θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
-# θ = positive.([1.1, 0.1, 0.01, 0.001]) 
 anim = Animation()
-opt = Optimise.ADAGrad(0.5)
-for i in 1:30
-    grads = only(Zygote.gradient(loss, θ)) # We compute the gradients given the kernel parameters and regularization
+for i in 1:25
+    grads = only((Zygote.gradient(loss, θ))) # We compute the gradients given the kernel parameters and regularization
     Optimise.update!(opt, θ, grads)
     scatter(
         x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
@@ -97,6 +108,9 @@ for i in 1:30
 end
 gif(anim)
 
+# Final loss
+loss(θ)
+
 # ### ParameterHandling.jl
 # Alternatively, we can use the [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl) package 
 # to handle the requirement that all kernel parameters should be positive. 
@@ -130,27 +144,30 @@ initial_θ = ParameterHandling.value(raw_initial_θ)
 
 # The loss with our starting point :
 
-loss(initial_θ)
+(loss ∘ unflatten)(flat_θ)
 
 # ## Training the model
 
-anim = Animation()
+# ### Cost per step
+
+@benchmark let θt = flat_θ[:], optt = Optimise.ADAGrad(0.5)
+    grads = (Zygote.gradient(loss ∘ unflatten, θt))[1] # We compute the gradients given the kernel parameters and regularization
+    Optimise.update!(optt, θt, grads)
+end
+
 opt = Optimise.ADAGrad(0.5)
-for i in 1:30
-    grads = only(Zygote.gradient(loss ∘ unflatten, flat_θ)) # We compute the gradients given the kernel parameters and regularization
+for i in 1:25
+    grads = (Zygote.gradient(loss ∘ unflatten, flat_θ))[1] # We compute the gradients given the kernel parameters and regularization
     Optimise.update!(opt, flat_θ, grads)
-    scatter(
-        x_train, y_train; lab="data", title="i = $(i), Loss = $(round((loss ∘ unflatten)(flat_θ), digits = 4))"
-    )
-    plot!(x_test, sinc; lab="true function")
-    plot!(x_test, f(x_test, x_train, y_train, unflatten(flat_θ)); lab="Prediction", lw=3.0)
-    frame(anim)
 end
-gif(anim)
+
+# Final loss
+
+(loss ∘ unflatten)(flat_θ)
 
 
 # ## Method 2: Functor
-# An alternative method is to use tools from Flux.jl, which is a fairly heavy package. 
+# An alternative method is to use tools from Flux.jl.
 
 # raw_initial_θ = (
 #     k1 = positive(1.1),
@@ -158,12 +175,62 @@ gif(anim)
 #     k3 = positive(0.01),
 #     noise_var=positive(0.001),
 # )
-k1 = 1.1
-k2 = 0.1
-k3 = 0.01
-noise_var = 0.001
+k1 = [1.1]
+k2 = [0.1]
+k3 = [0.01]
+noise_var = log.([0.001])
 
-kernel = (k1 * SqExponentialKernel() + k2 * Matern32Kernel()) ∘
-           ScaleTransform(k3)
+kernel = (ScaledKernel(SqExponentialKernel(), relu.(k1)) + ScaledKernel(Matern32Kernel(), k2)) ∘
+    ScaleTransform(map(exp,k3))
+
+θ = Flux.params(k1, k2, k3, noise_var)
+
+# kernel = (ScaledKernel(SqExponentialKernel(), softplus(θ[1])) + ScaledKernel(Matern32Kernel(), θ[2])) ∘
+#     ScaleTransform(θ[3])
+
+# This next 
+
+# function loss2()
+#     ŷ = kernelmatrix(kernel, x_train, x_train) * ((kernelmatrix(kernel, x_train) + θ[4][1] * I) \ y_train)
+#     return sum(abs2, y_train - ŷ) + θ[4][1] * norm(ŷ)
+# end
+
+function loss()
+    ŷ = kernelmatrix(kernel, x_train, x_train) * ((kernelmatrix(kernel, x_train)) \ y_train)
+    return sum(abs2, y_train - ŷ) + only(exp.(noise_var) .* norm(ŷ))
+end
+
+function f(x, x_train, y_train)
+    return kernelmatrix(kernel, x, x_train) *
+           ((kernelmatrix(kernel, x_train) + only(exp.(noise_var)) * I) \ y_train)
+end
+
+
+grads = Flux.gradient(loss, θ)
+for p in θ
+    println(grads[p])
+end
+
+
+grads = Flux.gradient(loss, θ)
+
+η = 0.1 # Learning Rate
+opt = Optimise.ADAGrad(η)
+# for p in θ
+#   update!(p, η * grads[p])
+# end
+
+anim = Animation()
+for i in 1:25
+    Optimise.update!(opt, θ, grads)
+    println(θ)
+
+    scatter(
+        x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(), digits = 4))"
+    )
+    plot!(x_test, sinc; lab="true function")
+    plot!(x_test, f(x_test, x_train, y_train); lab="Prediction", lw=3.0)
+    frame(anim)
+end
 
-Θ = Flux.params(k1, k2, k3) 
\ No newline at end of file
+gif(anim)
\ No newline at end of file

From 0fb3d7f8a879d1026ebb97879799af622863b30f Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 15:03:58 +0000
Subject: [PATCH 12/41] Add Flux.destructure example

---
 examples/train-kernel-parameters/script.jl | 187 ++++++++++-----------
 1 file changed, 89 insertions(+), 98 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 6db7cb81e..fe321bccb 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -2,7 +2,7 @@
 
 # In this example we show the two main methods to perform regression on a kernel from KernelFunctions.jl.
 
-# ## We load KernelFunctions and some other packages
+# We load KernelFunctions and some other packages
 
 using KernelFunctions
 using LinearAlgebra
@@ -14,8 +14,7 @@ using Flux
 using Flux: Optimise
 using Zygote
 using Random: seed!
-seed!(42)
-using ParameterHandling
+seed!(42);
 
 # ## Data Generation
 # We generated data in 1 dimension
@@ -26,30 +25,29 @@ N = 50 # Number of samples
 x_train = rand(Uniform(xmin, xmax), N) # We sample 100 random samples
 σ = 0.1
 y_train = sinc.(x_train) + randn(N) * σ # We create a function and add some noise
-x_test = range(xmin - 0.1, xmax + 0.1; length=300)
-
+x_test = range(xmin - 0.1, xmax + 0.1; length=300);
 # Plot the data
 
-scatter(x_train, y_train; lab="data")
-plot!(x_test, sinc; lab="true function")
+## scatter(x_train, y_train; lab="data")
+## plot!(x_test, sinc; lab="true function")
 
-# ## Method 1
-# The first method is to rebuild the parametrized kernel from a vector of parameters 
-# in each evaluation of the cost fuction. This is similar to the approach taken in 
-# [Stheno.jl](https://github.com/JuliaGaussianProcesses/Stheno.jl).
 
 
-# ### Base Approach
-# A simple way to ensure that the kernel parameters are positive
-# is to optimize over the logarithm of the parameters. 
+
+# ## Base Approach
+# The first option is to rebuild the parametrized kernel from a vector of parameters 
+# in each evaluation of the cost fuction. This is similar to the approach taken in 
+# [Stheno.jl](https://github.com/JuliaGaussianProcesses/Stheno.jl).
 
 # To train the kernel parameters via ForwardDiff.jl
 # we need to create a function creating a kernel from an array.
+# A simple way to ensure that the kernel parameters are positive
+# is to optimize over the logarithm of the parameters. 
 
 function kernelcall(θ)
     return (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘
            ScaleTransform(exp(θ[3]))
-end
+end;
 
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant
@@ -58,16 +56,15 @@ function f(x, x_train, y_train, θ)
     k = kernelcall(θ[1:3])
     return kernelmatrix(k, x, x_train) *
            ((kernelmatrix(k, x_train) + exp(θ[4]) * I) \ y_train)
-end
+end;
 
 # We look how the prediction looks like
 # with starting parameters [1.0, 1.0, 1.0, 1.0] we get :
 
-ŷ = f(x_test, x_train, y_train, log.(ones(4)))
-scatter(x_train, y_train; lab="data")
-plot!(x_test, sinc; lab="true function")
-plot!(x_test, ŷ; lab="prediction")
-
+ŷ = f(x_test, x_train, y_train, log.(ones(4)));
+## scatter(x_train, y_train; lab="data")
+## plot!(x_test, sinc; lab="true function")
+## plot!(x_test, ŷ; lab="prediction")
 
 # We define the loss based on the L2 norm both
 # for the loss and the regularization
@@ -75,46 +72,50 @@ plot!(x_test, ŷ; lab="prediction")
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, θ)
     return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
-end
-
-# ## Training the model
+end;
 
+# ### Training
+# Setting an initial value and initializing the optimizer: 
 θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
-opt = Optimise.ADAGrad(0.5)
+opt = Optimise.ADAGrad(0.5);
 
-# The loss with our starting point :
+# The loss with our starting point:
 
 loss(θ)
 
-# Cost for one step
+# Computational cost for one step
 
 @benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = only((Zygote.gradient(loss, θt))) # We compute the gradients given the kernel parameters and regularization
+    grads = only((Zygote.gradient(loss, θt))) 
     Optimise.update!(optt, θt, grads)
 end
 
-# The optimization 
+# Optimizing
 
-anim = Animation()
+## anim = Animation()
 for i in 1:25
-    grads = only((Zygote.gradient(loss, θ))) # We compute the gradients given the kernel parameters and regularization
+    grads = only((Zygote.gradient(loss, θ))) 
     Optimise.update!(opt, θ, grads)
-    scatter(
-        x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
-    )
-    plot!(x_test, sinc; lab="true function")
-    plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
-    frame(anim)
-end
-gif(anim)
+end;
+    ## scatter(
+    ##     x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
+    ## )
+    ## plot!(x_test, sinc; lab="true function")
+    ## plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
+    ## frame(anim)
+## end
+## gif(anim)
 
 # Final loss
 loss(θ)
 
-# ### ParameterHandling.jl
+
+# ## Using ParameterHandling.jl
 # Alternatively, we can use the [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl) package 
 # to handle the requirement that all kernel parameters should be positive. 
 
+using ParameterHandling
+
 raw_initial_θ = (
     k1 = positive(1.1),
     k2 = positive(0.1),
@@ -127,20 +128,20 @@ flat_θ, unflatten = ParameterHandling.value_flatten(raw_initial_θ);
 function kernelcall(θ)
     return (θ.k1 * SqExponentialKernel() + θ.k2 * Matern32Kernel()) ∘
            ScaleTransform(θ.k3)
-end
+end;
 
 function f(x, x_train, y_train, θ)
     k = kernelcall(θ)
     return kernelmatrix(k, x, x_train) *
            ((kernelmatrix(k, x_train) + θ.noise_var * I) \ y_train)
-end
+end;
 
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, θ)
     return sum(abs2, y_train - ŷ) + θ.noise_var * norm(ŷ)
-end
+end;
 
-initial_θ = ParameterHandling.value(raw_initial_θ)
+initial_θ = ParameterHandling.value(raw_initial_θ);
 
 # The loss with our starting point :
 
@@ -151,86 +152,76 @@ initial_θ = ParameterHandling.value(raw_initial_θ)
 # ### Cost per step
 
 @benchmark let θt = flat_θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = (Zygote.gradient(loss ∘ unflatten, θt))[1] # We compute the gradients given the kernel parameters and regularization
+    grads = (Zygote.gradient(loss ∘ unflatten, θt))[1] 
     Optimise.update!(optt, θt, grads)
 end
 
+# ### Complete optimization
+
 opt = Optimise.ADAGrad(0.5)
 for i in 1:25
-    grads = (Zygote.gradient(loss ∘ unflatten, flat_θ))[1] # We compute the gradients given the kernel parameters and regularization
+    grads = (Zygote.gradient(loss ∘ unflatten, flat_θ))[1] 
     Optimise.update!(opt, flat_θ, grads)
-end
+end;
 
 # Final loss
 
 (loss ∘ unflatten)(flat_θ)
 
 
-# ## Method 2: Functor
-# An alternative method is to use tools from Flux.jl.
+# ## Flux.destructure
+# If don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function. 
+# Again, we need to ensure that the parameters are positive. Note that the `exp` function now has to be in a different position. 
 
-# raw_initial_θ = (
-#     k1 = positive(1.1),
-#     k2 = positive(0.1),
-#     k3 = positive(0.01),
-#     noise_var=positive(0.001),
-# )
-k1 = [1.1]
-k2 = [0.1]
-k3 = [0.01]
-noise_var = log.([0.001])
 
-kernel = (ScaledKernel(SqExponentialKernel(), relu.(k1)) + ScaledKernel(Matern32Kernel(), k2)) ∘
-    ScaleTransform(map(exp,k3))
+θ = [1.1, 0.1, 0.01, 0.001] 
 
-θ = Flux.params(k1, k2, k3, noise_var)
+kernel = (θ[1] * SqExponentialKernel() + θ[2] * Matern32Kernel()) ∘
+ScaleTransform(θ[3])
 
-# kernel = (ScaledKernel(SqExponentialKernel(), softplus(θ[1])) + ScaledKernel(Matern32Kernel(), θ[2])) ∘
-#     ScaleTransform(θ[3])
+p, kernelc = Flux.destructure(kernel);
 
-# This next 
+# From theory we know the prediction for a test set x given
+# the kernel parameters and normalization constant
 
-# function loss2()
-#     ŷ = kernelmatrix(kernel, x_train, x_train) * ((kernelmatrix(kernel, x_train) + θ[4][1] * I) \ y_train)
-#     return sum(abs2, y_train - ŷ) + θ[4][1] * norm(ŷ)
-# end
+function f(x, x_train, y_train, θ)
+    k = kernelc(θ[1:3])
+    return kernelmatrix(k, x, x_train) *
+           ((kernelmatrix(k, x_train) + (θ[4]) * I) \ y_train)
+end;
 
-function loss()
-    ŷ = kernelmatrix(kernel, x_train, x_train) * ((kernelmatrix(kernel, x_train)) \ y_train)
-    return sum(abs2, y_train - ŷ) + only(exp.(noise_var) .* norm(ŷ))
-end
 
-function f(x, x_train, y_train)
-    return kernelmatrix(kernel, x, x_train) *
-           ((kernelmatrix(kernel, x_train) + only(exp.(noise_var)) * I) \ y_train)
-end
+# We define the loss based on the L2 norm both
+# for the loss and the regularization
 
+function loss(θ)
+    ŷ = f(x_train, x_train, y_train, exp.(θ))
+    return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
+end;
 
-grads = Flux.gradient(loss, θ)
-for p in θ
-    println(grads[p])
-end
+# ## Training the model
+
+# The loss with our starting point :
+θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
+loss(θ)
 
+# Initialize optimizer 
 
-grads = Flux.gradient(loss, θ)
+opt = Optimise.ADAGrad(0.5)
 
-η = 0.1 # Learning Rate
-opt = Optimise.ADAGrad(η)
-# for p in θ
-#   update!(p, η * grads[p])
-# end
+# Cost for one step
+
+@benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
+    grads = only((Zygote.gradient(loss, θt))) # We compute the gradients given the kernel parameters and regularization
+    Optimise.update!(optt, θt, grads)
+end
+
+# The optimization 
 
-anim = Animation()
 for i in 1:25
+    grads = only((Zygote.gradient(loss, θ))) # We compute the gradients given the kernel parameters and regularization
     Optimise.update!(opt, θ, grads)
-    println(θ)
-
-    scatter(
-        x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(), digits = 4))"
-    )
-    plot!(x_test, sinc; lab="true function")
-    plot!(x_test, f(x_test, x_train, y_train); lab="Prediction", lw=3.0)
-    frame(anim)
-end
+end;
 
-gif(anim)
\ No newline at end of file
+# Final loss
+loss(θ)

From 6907d712d7bec3ccaf05afd78f1b6d73a5f621ed Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 15:13:44 +0000
Subject: [PATCH 13/41] Add some nothing

---
 examples/train-kernel-parameters/script.jl | 45 ++++++++++++++--------
 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index fe321bccb..7415a5c2f 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -25,7 +25,8 @@ N = 50 # Number of samples
 x_train = rand(Uniform(xmin, xmax), N) # We sample 100 random samples
 σ = 0.1
 y_train = sinc.(x_train) + randn(N) * σ # We create a function and add some noise
-x_test = range(xmin - 0.1, xmax + 0.1; length=300);
+x_test = range(xmin - 0.1, xmax + 0.1; length=300)
+nothing #hide
 # Plot the data
 
 ## scatter(x_train, y_train; lab="data")
@@ -47,7 +48,8 @@ x_test = range(xmin - 0.1, xmax + 0.1; length=300);
 function kernelcall(θ)
     return (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘
            ScaleTransform(exp(θ[3]))
-end;
+end
+nothing #hide
 
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant
@@ -56,15 +58,17 @@ function f(x, x_train, y_train, θ)
     k = kernelcall(θ[1:3])
     return kernelmatrix(k, x, x_train) *
            ((kernelmatrix(k, x_train) + exp(θ[4]) * I) \ y_train)
-end;
+end
+nothing #hide
 
 # We look how the prediction looks like
 # with starting parameters [1.0, 1.0, 1.0, 1.0] we get :
 
-ŷ = f(x_test, x_train, y_train, log.(ones(4)));
+ŷ = f(x_test, x_train, y_train, log.(ones(4)))
 ## scatter(x_train, y_train; lab="data")
 ## plot!(x_test, sinc; lab="true function")
 ## plot!(x_test, ŷ; lab="prediction")
+nothing #hide
 
 # We define the loss based on the L2 norm both
 # for the loss and the regularization
@@ -72,12 +76,14 @@ ŷ = f(x_test, x_train, y_train, log.(ones(4)));
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, θ)
     return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
-end;
+end
+nothing #hide
 
 # ### Training
 # Setting an initial value and initializing the optimizer: 
 θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
-opt = Optimise.ADAGrad(0.5);
+opt = Optimise.ADAGrad(0.5)
+nothing #hide
 
 # The loss with our starting point:
 
@@ -123,25 +129,30 @@ raw_initial_θ = (
     noise_var=positive(0.001),
 )
 
-flat_θ, unflatten = ParameterHandling.value_flatten(raw_initial_θ);
+flat_θ, unflatten = ParameterHandling.value_flatten(raw_initial_θ)
+nothing #hide
 
 function kernelcall(θ)
     return (θ.k1 * SqExponentialKernel() + θ.k2 * Matern32Kernel()) ∘
            ScaleTransform(θ.k3)
-end;
+end
+nothing #hide
 
 function f(x, x_train, y_train, θ)
     k = kernelcall(θ)
     return kernelmatrix(k, x, x_train) *
            ((kernelmatrix(k, x_train) + θ.noise_var * I) \ y_train)
-end;
+end
+nothing #hide
 
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, θ)
     return sum(abs2, y_train - ŷ) + θ.noise_var * norm(ŷ)
-end;
+end
+nothing #hide
 
-initial_θ = ParameterHandling.value(raw_initial_θ);
+initial_θ = ParameterHandling.value(raw_initial_θ)
+nothing #hide
 
 # The loss with our starting point :
 
@@ -162,7 +173,8 @@ opt = Optimise.ADAGrad(0.5)
 for i in 1:25
     grads = (Zygote.gradient(loss ∘ unflatten, flat_θ))[1] 
     Optimise.update!(opt, flat_θ, grads)
-end;
+end
+nothing #hide
 
 # Final loss
 
@@ -188,7 +200,8 @@ function f(x, x_train, y_train, θ)
     k = kernelc(θ[1:3])
     return kernelmatrix(k, x, x_train) *
            ((kernelmatrix(k, x_train) + (θ[4]) * I) \ y_train)
-end;
+end
+nothing #hide
 
 
 # We define the loss based on the L2 norm both
@@ -197,7 +210,8 @@ end;
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, exp.(θ))
     return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
-end;
+end
+nothing #hide
 
 # ## Training the model
 
@@ -221,7 +235,8 @@ end
 for i in 1:25
     grads = only((Zygote.gradient(loss, θ))) # We compute the gradients given the kernel parameters and regularization
     Optimise.update!(opt, θ, grads)
-end;
+end
+nothing #hide
 
 # Final loss
 loss(θ)

From 98c6913027b7a04f9ec1ff8b7b79d6528fceeb4d Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 15:37:53 +0000
Subject: [PATCH 14/41] Squashed commit of the following:
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

commit f9bbd84beb487c8335adf55034cd78a1279681c1
Author: st-- <st--@users.noreply.github.com>
Date:   Fri Jan 28 09:11:50 2022 +0100

    make nystrom work with AbstractVector (#427)

    * make nystrom work with AbstractVector

    * add test

    * Update test/approximations/nystrom.jl

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

    * patch bump

    * Update test/approximations/nystrom.jl

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

    * Apply suggestions from code review

    Co-authored-by: David Widmann <devmotion@users.noreply.github.com>

    * Apply suggestions from code review

    * deprecate

    * Apply suggestions from code review

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

    * Apply suggestions from code review

    Co-authored-by: David Widmann <devmotion@users.noreply.github.com>

    * Apply suggestions from code review

    Co-authored-by: Théo Galy-Fajou <theo.galyfajou@gmail.com>

    * Update src/approximations/nystrom.jl

    Co-authored-by: Théo Galy-Fajou <theo.galyfajou@gmail.com>

    * Update src/approximations/nystrom.jl

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
    Co-authored-by: David Widmann <devmotion@users.noreply.github.com>
    Co-authored-by: Théo Galy-Fajou <theo.galyfajou@gmail.com>

commit d1c68a9dc816496ea500b4d8e37ee948d4150cec
Author: st-- <st--@users.noreply.github.com>
Date:   Thu Jan 13 22:33:43 2022 +0100

    fix Distances compat (#423)

    * CompatHelper: bump compat for Distances to 0.10 for package test, (keep existing compat)

    * try out Theo's fix

    * fix test compat

    * use ForwardDiff for chain rule test of SqMahalanobis

    * test on 1.4 instead of 1.3 - see if the chainrules test passes there

    * revert version branch

    * revert to 1.3

    * test_broken for older Julia versions

    Co-authored-by: CompatHelper Julia <compathelper_noreply@julialang.org>

commit 93d33c2032b86d87e05eaa750101bf4c4e5ef1f8
Author: st-- <st--@users.noreply.github.com>
Date:   Wed Jan 12 14:11:14 2022 +0100

    fix figure & cleanup (#422)

    * fix figure & cleanup

    * bump LIBSVM compat & Manifest

    * improve writing, replaces #321

commit 40cb59e1463fee628dfd097b5e6a6bcd26fe50e2
Author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Date:   Wed Jan 12 09:39:01 2022 +0200

    CompatHelper: bump compat for Kronecker to 0.5 for package docs, (keep existing compat) (#367)

    * CompatHelper: bump compat for Kronecker to 0.5 for package docs, (keep existing compat)

    * ] up

    Co-authored-by: CompatHelper Julia <compathelper_noreply@julialang.org>
    Co-authored-by: st-- <st--@users.noreply.github.com>

commit 720452921ecae18b6960839d57bd524e20601e94
Author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Date:   Tue Jan 11 18:37:23 2022 +0200

    CompatHelper: bump compat for Kronecker to 0.5 for package test, (keep existing compat) (#366)

    Co-authored-by: CompatHelper Julia <compathelper_noreply@julialang.org>
    Co-authored-by: st-- <st--@users.noreply.github.com>

commit 924925d8e9bc74f93c31ce2ffd19a1ceb74d5dd4
Author: st-- <st--@users.noreply.github.com>
Date:   Tue Jan 11 16:26:02 2022 +0100

    switch SVM example to half-moon dataset (#421)

commit 992b665d0391e60c3bad713e8ebeaef3c4e297ba
Author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Date:   Fri Dec 24 12:18:56 2021 +0200

    CompatHelper: bump compat for SpecialFunctions to 2 for package test, (keep existing compat) (#412)

    Co-authored-by: CompatHelper Julia <compathelper_noreply@julialang.org>
    Co-authored-by: Théo Galy-Fajou <theo.galyfajou@gmail.com>
    Co-authored-by: st-- <st--@users.noreply.github.com>

commit 04fa7f73d0b9c61a53710818c6dca36e474e7b3e
Author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Date:   Thu Dec 23 13:33:59 2021 +0200

    CompatHelper: bump compat for SpecialFunctions to 2, (keep existing compat) (#411)

    Co-authored-by: CompatHelper Julia <compathelper_noreply@julialang.org>
    Co-authored-by: Théo Galy-Fajou <theo.galyfajou@gmail.com>
    Co-authored-by: st-- <st--@users.noreply.github.com>

commit c0fc3e139ec8a57b2c318db276971c13fa6954b9
Author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Date:   Thu Dec 23 01:10:40 2021 +0200

    CompatHelper: add new compat entry for Compat at version 3 for package test, (keep existing compat) (#418)

    Co-authored-by: CompatHelper Julia <compathelper_noreply@julialang.org>

commit 05fe34077dffe66ed2808c07e21e261496c4f089
Author: st-- <st--@users.noreply.github.com>
Date:   Tue Dec 21 00:49:37 2021 +0200

    use only() instead of first() (#403)

    * use only() instead of first() for 1-"vectors" that were for the benefit of Flux

    * fix one test that should not have worked as it was

    * add missing scalar Sinus constructor

commit 2d172125e72ca932133266c3b23d0865bec539df
Author: st-- <st--@users.noreply.github.com>
Date:   Sat Dec 18 23:43:30 2021 +0200

    Zygote AD failure workarounds & test cleanup (#414)

    Zygote AD failures:

    * revert #409 (test_utils workaround for broken Zygote - now working again)

    * disable broken Zygote AD test for ChainTransform

    Improved tests:

    * finer-grained testsets

    * add missing test cases to test_AD

    * replace test_FiniteDiff with test_AD(..., :FiniteDiff, ...)

    * remove code duplication

commit 3c499497b55fbc4b17ae5edd0a3ba24e73346d4f
Author: Théo Galy-Fajou <theo.galyfajou@gmail.com>
Date:   Wed Nov 24 18:32:19 2021 +0100

    Fix typo in valid_inputs error (#408)

    * Fix typo in valid_inputs error

    * Update src/utils.jl

    Co-authored-by: David Widmann <devmotion@users.noreply.github.com>

    Co-authored-by: David Widmann <devmotion@users.noreply.github.com>

commit 9955044b4d82e4732d0f9a78d67671e02ee9053f
Author: st-- <st--@users.noreply.github.com>
Date:   Wed Nov 24 18:55:18 2021 +0200

    Fix for Zygote 0.6.30 breaking our tests (#409)

    * restrict Zygote to <0.6.30

    * revert Zygote test restriction and add finer-grained testset

    * Update test/utils.jl

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

    * revert testset

    * mark test_broken

    * Use `@test_throws` instead of `@test_broken`

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
    Co-authored-by: David Widmann <devmotion@users.noreply.github.com>

commit 33d64d1425a4aa8748bd8ca453ca5be41eb53280
Author: Théo Galy-Fajou <theo.galyfajou@gmail.com>
Date:   Thu Nov 4 14:23:57 2021 +0100

    Add benchmarking CI (#399)

    * Add benchmark file

    * delete old benchmarks

    * Add github action

    * Add Project

    * Apply suggestions from code review

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

    * Missing end of line

    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

commit 360ce1007a106abc3b25f3bfa9cbe9846355b4b5
Author: David Widmann <devmotion@users.noreply.github.com>
Date:   Tue Nov 2 11:09:58 2021 +0100

    Update docstring of `GibbsKernel` (#395)
---
 .github/workflows/benchmark.yml               |  24 ++
 .gitignore                                    |   1 +
 Project.toml                                  |   4 +-
 benchmark/MLKernels.jl                        |  22 --
 benchmark/Project.toml                        |   3 +
 benchmark/benchmarks.jl                       |  55 +++-
 benchmark/kernelmatrix.jl                     |  39 ---
 docs/Manifest.toml                            | 133 ++++----
 docs/Project.toml                             |   2 +-
 examples/support-vector-machine/Manifest.toml | 306 ++++++++++--------
 examples/support-vector-machine/Project.toml  |   2 +-
 examples/support-vector-machine/script.jl     |  66 ++--
 src/approximations/nystrom.jl                 |  56 ++--
 src/basekernels/constant.jl                   |   4 +-
 src/basekernels/exponential.jl                |   4 +-
 src/basekernels/fbm.jl                        |   6 +-
 src/basekernels/matern.jl                     |   4 +-
 src/basekernels/polynomial.jl                 |  10 +-
 src/basekernels/rational.jl                   |  16 +-
 src/chainrules.jl                             |   2 +-
 src/distances/sinus.jl                        |   4 +-
 src/kernels/gibbskernel.jl                    |  18 +-
 src/kernels/scaledkernel.jl                   |   4 +-
 src/kernels/transformedkernel.jl              |   4 +-
 src/transform/ardtransform.jl                 |   2 +-
 src/transform/periodic_transform.jl           |   8 +-
 src/transform/scaletransform.jl               |  12 +-
 src/utils.jl                                  |   2 +-
 test/Project.toml                             |   8 +-
 test/approximations/nystrom.jl                |  10 +-
 test/basekernels/constant.jl                  |   2 +-
 test/basekernels/exponential.jl               |   2 +-
 test/chainrules.jl                            |   8 +-
 test/distances/sinus.jl                       |   3 +-
 test/runtests.jl                              |   1 +
 test/test_utils.jl                            | 162 +++-------
 test/transform/chaintransform.jl              |   4 +-
 37 files changed, 527 insertions(+), 486 deletions(-)
 create mode 100644 .github/workflows/benchmark.yml
 delete mode 100644 benchmark/MLKernels.jl
 create mode 100644 benchmark/Project.toml
 delete mode 100644 benchmark/kernelmatrix.jl

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 000000000..3dfd39d4c
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,24 @@
+name: Run benchmarks
+
+on:
+  pull_request:
+
+jobs:
+  Benchmark:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'performance critical')
+    env:
+      JULIA_DEBUG: BenchmarkCI
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: 1.6
+      - name: Install dependencies
+        run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI"'
+      - name: Run benchmarks
+        run: julia -e "using BenchmarkCI; BenchmarkCI.judge()"
+      - name: Post results
+        run: julia -e "using BenchmarkCI; BenchmarkCI.postjudge()"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index eefb4afce..ba19ae215 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@
 /test/Manifest.toml
 coverage/
 .DS_store
+benchmark/Manifest.toml
diff --git a/Project.toml b/Project.toml
index d3a56f055..822c5ee2d 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,6 +1,6 @@
 name = "KernelFunctions"
 uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
-version = "0.10.26"
+version = "0.10.28"
 
 [deps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
@@ -30,7 +30,7 @@ Functors = "0.1, 0.2"
 IrrationalConstants = "0.1"
 LogExpFunctions = "0.2.1, 0.3"
 Requires = "1.0.1"
-SpecialFunctions = "0.8, 0.9, 0.10, 1"
+SpecialFunctions = "0.8, 0.9, 0.10, 1, 2"
 StatsBase = "0.32, 0.33"
 TensorCore = "0.1"
 ZygoteRules = "0.2"
diff --git a/benchmark/MLKernels.jl b/benchmark/MLKernels.jl
deleted file mode 100644
index 1b678bdd7..000000000
--- a/benchmark/MLKernels.jl
+++ /dev/null
@@ -1,22 +0,0 @@
-using MLKernels
-
-SUITE["MLKernels"] = BenchmarkGroup()
-
-mlkernelnames = ["SquaredExponentialKernel"]
-kernels = Dict{String,MLKernels.Kernel}()
-for k in mlkernelnames
-    SUITE["MLKernels"][k] = BenchmarkGroup()
-    kernels[k] = eval(Meta.parse("MLKernels." * k * "(alpha)"))
-end
-
-for k in mlkernelnames
-    SUITE["MLKernels"][k]["k(X,Y)"] = @benchmarkable MLKernels.kernelmatrix(
-        $(kernels[k]), $X, $Y
-    )
-    # SUITE["MLKernels"][k][kt]["k!(X,Y)"] = @benchmarkable MLKernels.kernelmatrix!(KXY,$(kernels[k][kt]),$X,$Y) setup=(KXY=copy($KXY))
-    SUITE["MLKernels"][k]["k(X)"] = @benchmarkable MLKernels.kernelmatrix($(kernels[k]), $X)
-    # SUITE["MLKernels"][k][kt]["k!(X)"] = @benchmarkable MLKernels.kernelmatrix!(KX,$(kernels[k][kt]),$X) setup=(KX=copy($KX))
-    # SUITE["MLKernels"][k][kt]["kdiag(X)"] = @benchmarkable MLKernels.kernelmatrix_diag($(kernels[k][kt]),$X)
-    # SUITE["MLKernels"][k][kt]["kdiag!(X)"] = @benchmarkable MLKernels.kernelmatrix_diag!(kX,$(kernels[k][kt]),$X) setup=(kX=copy($kX))
-end
-# results = run(SUITE)
diff --git a/benchmark/Project.toml b/benchmark/Project.toml
new file mode 100644
index 000000000..cf9c7b9c1
--- /dev/null
+++ b/benchmark/Project.toml
@@ -0,0 +1,3 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 17181b365..0cb80e5ce 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -1,23 +1,46 @@
 using BenchmarkTools
-using Random
-using Distances, LinearAlgebra
+using KernelFunctions
 
-const SUITE = BenchmarkGroup()
+N1 = 10
+N2 = 20
 
-Random.seed!(1234)
+X = rand(N1, N2)
+Xc = ColVecs(X)
+Xr = RowVecs(X)
+Xv = collect.(eachcol(X))
+Y = rand(N1, N2)
+Yc = ColVecs(Y)
+Yr = RowVecs(Y)
+Yv = collect.(eachcol(Y))
 
-dim = 50
-N1 = 1000;
-N2 = 500;
-alpha = 2.0
+# Create the general suite of benchmarks
+SUITE = BenchmarkGroup()
 
-X = rand(Float64, N1, dim)
-Y = rand(Float64, N2, dim)
+kernels = Dict(
+    "SqExponential" => SqExponentialKernel(), "Exponential" => ExponentialKernel()
+)
 
-KXY = rand(Float64, N1, N2)
-KX = rand(Float64, N1, N1)
-sKX = Symmetric(rand(Float64, N1, N1))
-kX = rand(Float64, N1)
+inputtypes = Dict("ColVecs" => (Xc, Yc), "RowVecs" => (Xr, Yr), "Vecs" => (Xv, Yv))
 
-include("kernelmatrix.jl")
-include("MLKernels.jl")
+functions = Dict(
+    "kernelmatrixX" => (kernel, X, Y) -> kernelmatrix(kernel, X),
+    "kernelmatrixXY" => (kernel, X, Y) -> kernelmatrix(kernel, X, Y),
+    "kernelmatrix_diagX" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X),
+    "kernelmatrix_diagXY" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X, Y),
+)
+
+for (kname, kernel) in kernels
+    SUITE[kname] = sk = BenchmarkGroup()
+    for (inputname, (X, Y)) in inputtypes
+        sk[inputname] = si = BenchmarkGroup()
+        for (fname, f) in functions
+            si[fname] = @benchmarkable $f($kernel, $X, $Y)
+        end
+    end
+end
+
+# Uncomment the following to run benchmark locally
+
+# tune!(SUITE)
+
+# results = run(SUITE, verbose=true)
diff --git a/benchmark/kernelmatrix.jl b/benchmark/kernelmatrix.jl
deleted file mode 100644
index e8e708012..000000000
--- a/benchmark/kernelmatrix.jl
+++ /dev/null
@@ -1,39 +0,0 @@
-using KernelFunctions
-
-SUITE["KernelFunctions"] = BenchmarkGroup()
-
-kernelnames = ["SqExponentialKernel"]
-kerneltypes = ["ARD", "ISO"]
-kernels = Dict{String,Dict{String,KernelFunctions.Kernel}}()
-for k in kernelnames
-    kernels[k] = Dict{String,KernelFunctions.Kernel}()
-    SUITE["KernelFunctions"][k] = BenchmarkGroup()
-    for kt in kerneltypes
-        SUITE["KernelFunctions"][k][kt] = BenchmarkGroup()
-        kernels[k][kt] = eval(
-            Meta.parse(
-                "KernelFunctions." *
-                k *
-                "(" *
-                (kt == "ARD" ? "alpha*ones(Float64,dim)" : "alpha") *
-                ")",
-            ),
-        )
-    end
-end
-
-for k in kernelnames
-    for kt in kerneltypes
-        SUITE["KernelFunctions"][k][kt]["k(X,Y)"] = @benchmarkable KernelFunctions.kernelmatrix(
-            $(kernels[k][kt]), $X, $Y; obsdim=1
-        )
-        # SUITE["KernelFunctions"][k][kt]["k!(X,Y)"] = @benchmarkable KernelFunctions.kernelmatrix!(KXY,$(kernels[k][kt]),$X,$Y) setup=(KXY=copy($KXY))
-        SUITE["KernelFunctions"][k][kt]["k(X)"] = @benchmarkable KernelFunctions.kernelmatrix(
-            $(kernels[k][kt]), $X; obsdim=1
-        )
-        # SUITE["KernelFunctions"][k][kt]["k!(X)"] = @benchmarkable KernelFunctions.kernelmatrix!(KX,$(kernels[k][kt]),$X) setup=(KX=copy($KX))
-        # SUITE["KernelFunctions"][k][kt]["kdiag(X)"] = @benchmarkable KernelFunctions.kernelmatrix_diag($(kernels[k][kt]),$X)
-        # SUITE["KernelFunctions"][k][kt]["kdiag!(X)"] = @benchmarkable KernelFunctions.kernelmatrix_diag!(kX,$(kernels[k][kt]),$X) setup=(kX=copy($kX))
-    end
-end
-# results = run(SUITE)
diff --git a/docs/Manifest.toml b/docs/Manifest.toml
index eb54b0b35..647ea24ad 100644
--- a/docs/Manifest.toml
+++ b/docs/Manifest.toml
@@ -6,10 +6,10 @@ uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9"
 version = "0.0.1"
 
 [[AbstractFFTs]]
-deps = ["LinearAlgebra"]
-git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0"
+deps = ["ChainRulesCore", "LinearAlgebra"]
+git-tree-sha1 = "6f1d9bc1c08f9f4a8fa92e3ea3cb50153a1b40d4"
 uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
-version = "1.0.1"
+version = "1.1.0"
 
 [[ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
@@ -22,15 +22,21 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[ChainRulesCore]]
 deps = ["Compat", "LinearAlgebra", "SparseArrays"]
-git-tree-sha1 = "f53ca8d41e4753c41cdafa6ec5f7ce914b34be54"
+git-tree-sha1 = "926870acb6cbcf029396f2f2de030282b6bc1941"
 uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-version = "0.10.13"
+version = "1.11.4"
+
+[[ChangesOfVariables]]
+deps = ["ChainRulesCore", "LinearAlgebra", "Test"]
+git-tree-sha1 = "bf98fa45a0a4cee295de98d4c1462be26345b9a1"
+uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
+version = "0.1.2"
 
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
-git-tree-sha1 = "344f143fa0ec67e47917848795ab19c6a455f32c"
+git-tree-sha1 = "44c37b4636bc54afac5c574d2d02b625349d6582"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "3.32.0"
+version = "3.41.0"
 
 [[CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
@@ -41,16 +47,22 @@ git-tree-sha1 = "455419f7e328a1a2493cabc6428d79e951349769"
 uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
 version = "0.1.1"
 
+[[CovarianceEstimation]]
+deps = ["LinearAlgebra", "Statistics", "StatsBase"]
+git-tree-sha1 = "a3e070133acab996660d31dcf479ea42849e368f"
+uuid = "587fd27a-f159-11e8-2dae-1979310e6154"
+version = "0.2.7"
+
 [[DataAPI]]
-git-tree-sha1 = "ee400abb2298bd13bfc3df1c412ed228061a2385"
+git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8"
 uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.7.0"
+version = "1.9.0"
 
 [[DataStructures]]
 deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677"
+git-tree-sha1 = "3daef5523dd2e769dad2365274f760ff5f282c7d"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.18.9"
+version = "0.18.11"
 
 [[Dates]]
 deps = ["Printf"]
@@ -61,10 +73,10 @@ deps = ["Mmap"]
 uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 
 [[Distances]]
-deps = ["LinearAlgebra", "Statistics", "StatsAPI"]
-git-tree-sha1 = "abe4ad222b26af3337262b8afb28fab8d215e9f8"
+deps = ["LinearAlgebra", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "3258d0659f812acde79e8a74b11f17ac06d0ca04"
 uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
-version = "0.10.3"
+version = "0.10.7"
 
 [[Distributed]]
 deps = ["Random", "Serialization", "Sockets"]
@@ -72,15 +84,15 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[DocStringExtensions]]
 deps = ["LibGit2"]
-git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
+git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b"
 uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
-version = "0.8.5"
+version = "0.8.6"
 
 [[Documenter]]
 deps = ["ANSIColoredPrinters", "Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
-git-tree-sha1 = "350dced36c11f794c6c4da5dc6493ec894e50c16"
+git-tree-sha1 = "f425293f7e0acaf9144de6d731772de156676233"
 uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
-version = "0.27.5"
+version = "0.27.10"
 
 [[Downloads]]
 deps = ["ArgTools", "LibCURL", "NetworkOptions"]
@@ -88,15 +100,14 @@ uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 
 [[FillArrays]]
 deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"]
-git-tree-sha1 = "8c8eac2af06ce35973c3eadb4ab3243076a408e7"
+git-tree-sha1 = "8756f9935b7ccc9064c6eef0bff0ad643df733a3"
 uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "0.12.1"
+version = "0.12.7"
 
 [[Functors]]
-deps = ["MacroTools"]
-git-tree-sha1 = "4cd9e70bf8fce05114598b663ad79dfe9ae432b3"
+git-tree-sha1 = "e4768c3b7f597d5a352afa09874d16e3c3f6ead2"
 uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
-version = "0.2.3"
+version = "0.2.7"
 
 [[IOCapture]]
 deps = ["Logging", "Random"]
@@ -108,10 +119,16 @@ version = "0.2.2"
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
+[[InverseFunctions]]
+deps = ["Test"]
+git-tree-sha1 = "a7254c0acd8e62f1ac75ad24d5db43f5f19f3c65"
+uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
+version = "0.1.2"
+
 [[IrrationalConstants]]
-git-tree-sha1 = "f76424439413893a832026ca355fe273e93bce94"
+git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151"
 uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
-version = "0.1.0"
+version = "0.1.1"
 
 [[JLLWrappers]]
 deps = ["Preferences"]
@@ -129,13 +146,13 @@ version = "0.21.2"
 deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Random", "Requires", "SpecialFunctions", "StatsBase", "TensorCore", "Test", "ZygoteRules"]
 path = ".."
 uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
-version = "0.10.12"
+version = "0.10.27"
 
 [[Kronecker]]
 deps = ["LinearAlgebra", "NamedDims", "SparseArrays", "StatsBase"]
-git-tree-sha1 = "9e34d39bda0ac9bc82ffbab735ef9c857e94605f"
+git-tree-sha1 = "a51f46415c844dee694cb8b20a3fcbe6dba342c2"
 uuid = "2c470bb0-bcc8-11e8-3dad-c9649493f05e"
-version = "0.4.4"
+version = "0.5.1"
 
 [[LibCURL]]
 deps = ["LibCURL_jll", "MozillaCACerts_jll"]
@@ -161,19 +178,19 @@ deps = ["Libdl"]
 uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [[LogExpFunctions]]
-deps = ["DocStringExtensions", "LinearAlgebra"]
-git-tree-sha1 = "7bd5f6565d80b6bf753738d2bc40a5dfea072070"
+deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"]
+git-tree-sha1 = "e5718a00af0ab9756305a0392832c8952c7426c1"
 uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
-version = "0.2.5"
+version = "0.3.6"
 
 [[Logging]]
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
 [[MacroTools]]
 deps = ["Markdown", "Random"]
-git-tree-sha1 = "0fb723cd8c45858c22169b2e42269e53271a6df7"
+git-tree-sha1 = "3d3e902b31198a27340d0bf00d6ac452866021cf"
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.5.7"
+version = "0.5.9"
 
 [[Markdown]]
 deps = ["Base64"]
@@ -185,9 +202,9 @@ uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
 
 [[Missings]]
 deps = ["DataAPI"]
-git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7"
+git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f"
 uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
-version = "1.0.0"
+version = "1.0.2"
 
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -196,14 +213,18 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
 
 [[NamedDims]]
-deps = ["AbstractFFTs", "LinearAlgebra", "Pkg", "Requires", "Statistics"]
-git-tree-sha1 = "52985b34519b12fd0dcebbe34e74b2dbe6d03183"
+deps = ["AbstractFFTs", "ChainRulesCore", "CovarianceEstimation", "LinearAlgebra", "Pkg", "Requires", "Statistics"]
+git-tree-sha1 = "88dce79529a358f6efd13225d131bec958a18f1d"
 uuid = "356022a1-0364-5f58-8944-0da4b18d706f"
-version = "0.2.35"
+version = "0.2.43"
 
 [[NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 
+[[OpenLibm_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
+
 [[OpenSpecFun_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
 git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
@@ -217,15 +238,15 @@ version = "1.4.1"
 
 [[PDMats]]
 deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
-git-tree-sha1 = "4dd403333bcf0909341cfe57ec115152f937d7d8"
+git-tree-sha1 = "ee26b350276c51697c9c2d88a072b339f9f03d73"
 uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
-version = "0.11.1"
+version = "0.11.5"
 
 [[Parsers]]
 deps = ["Dates"]
-git-tree-sha1 = "477bf42b4d1496b454c10cce46645bb5b8a0cf2c"
+git-tree-sha1 = "d7fa6237da8004be601e19bd6666083056649918"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "2.0.2"
+version = "2.1.3"
 
 [[Pkg]]
 deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
@@ -233,9 +254,9 @@ uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
+git-tree-sha1 = "2cf929d64681236a2e074ffafb8d568733d2e6af"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.2.2"
+version = "1.2.3"
 
 [[Printf]]
 deps = ["Unicode"]
@@ -251,9 +272,9 @@ uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
 [[Requires]]
 deps = ["UUIDs"]
-git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621"
+git-tree-sha1 = "8f82019e525f4d5c669692772a6f4b0a58b06a6a"
 uuid = "ae029012-a4dd-5104-9daa-d747884805df"
-version = "1.1.3"
+version = "1.2.0"
 
 [[SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
@@ -279,25 +300,25 @@ deps = ["LinearAlgebra", "Random"]
 uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [[SpecialFunctions]]
-deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"]
-git-tree-sha1 = "508822dca004bf62e210609148511ad03ce8f1d8"
+deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
+git-tree-sha1 = "e08890d19787ec25029113e88c34ec20cac1c91e"
 uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
-version = "1.6.0"
+version = "2.0.0"
 
 [[Statistics]]
 deps = ["LinearAlgebra", "SparseArrays"]
 uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [[StatsAPI]]
-git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
+git-tree-sha1 = "d88665adc9bcf45903013af0982e2fd05ae3d0a6"
 uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
-version = "1.0.0"
+version = "1.2.0"
 
 [[StatsBase]]
-deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
-git-tree-sha1 = "fed1ec1e65749c4d96fc20dd13bea72b55457e62"
+deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "51383f2d367eb3b444c961d485c565e4c0cf4ba0"
 uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
-version = "0.33.9"
+version = "0.33.14"
 
 [[SuiteSparse]]
 deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
@@ -334,9 +355,9 @@ uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
 
 [[ZygoteRules]]
 deps = ["MacroTools"]
-git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7"
+git-tree-sha1 = "8c1a8e4dfacb1fd631745552c8db35d0deb09ea0"
 uuid = "700de1a5-db45-46bc-99cf-38207098b444"
-version = "0.2.1"
+version = "0.2.2"
 
 [[nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]
diff --git a/docs/Project.toml b/docs/Project.toml
index 4f10fcafa..c5d584e87 100644
--- a/docs/Project.toml
+++ b/docs/Project.toml
@@ -7,6 +7,6 @@ PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150"
 [compat]
 Documenter = "0.27"
 KernelFunctions = "0.10"
-Kronecker = "0.4"
+Kronecker = "0.4, 0.5"
 PDMats = "0.11"
 julia = "1.3"
diff --git a/examples/support-vector-machine/Manifest.toml b/examples/support-vector-machine/Manifest.toml
index 56c5cb51c..f5cdc076d 100644
--- a/examples/support-vector-machine/Manifest.toml
+++ b/examples/support-vector-machine/Manifest.toml
@@ -2,9 +2,9 @@
 
 [[Adapt]]
 deps = ["LinearAlgebra"]
-git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7"
+git-tree-sha1 = "af92965fb30777147966f58acb05da51c5616b5f"
 uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
-version = "3.3.1"
+version = "3.3.3"
 
 [[ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
@@ -17,27 +17,33 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[Bzip2_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "c3598e525718abcc440f69cc6d5f60dda0a1b61e"
+git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2"
 uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
-version = "1.0.6+5"
+version = "1.0.8+0"
 
 [[Cairo_jll]]
 deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
-git-tree-sha1 = "e2f47f6d8337369411569fd45ae5753ca10394c6"
+git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2"
 uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
-version = "1.16.0+6"
+version = "1.16.1+1"
 
 [[ChainRulesCore]]
 deps = ["Compat", "LinearAlgebra", "SparseArrays"]
-git-tree-sha1 = "f53ca8d41e4753c41cdafa6ec5f7ce914b34be54"
+git-tree-sha1 = "926870acb6cbcf029396f2f2de030282b6bc1941"
 uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-version = "0.10.13"
+version = "1.11.4"
+
+[[ChangesOfVariables]]
+deps = ["ChainRulesCore", "LinearAlgebra", "Test"]
+git-tree-sha1 = "bf98fa45a0a4cee295de98d4c1462be26345b9a1"
+uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
+version = "0.1.2"
 
 [[ColorSchemes]]
 deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random"]
-git-tree-sha1 = "9995eb3977fbf67b86d0a0a0508e83017ded03f2"
+git-tree-sha1 = "a851fec56cb73cfdf43762999ec72eff5b86882a"
 uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
-version = "3.14.0"
+version = "3.15.0"
 
 [[ColorTypes]]
 deps = ["FixedPointNumbers", "Random"]
@@ -53,9 +59,9 @@ version = "0.12.8"
 
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
-git-tree-sha1 = "344f143fa0ec67e47917848795ab19c6a455f32c"
+git-tree-sha1 = "44c37b4636bc54afac5c574d2d02b625349d6582"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "3.32.0"
+version = "3.41.0"
 
 [[CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
@@ -73,15 +79,15 @@ uuid = "d38c429a-6771-53c6-b99e-75d170b6e991"
 version = "0.5.7"
 
 [[DataAPI]]
-git-tree-sha1 = "ee400abb2298bd13bfc3df1c412ed228061a2385"
+git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8"
 uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.7.0"
+version = "1.9.0"
 
 [[DataStructures]]
 deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "7d9d316f04214f7efdbb6398d545446e246eff02"
+git-tree-sha1 = "3daef5523dd2e769dad2365274f760ff5f282c7d"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.18.10"
+version = "0.18.11"
 
 [[DataValueInterfaces]]
 git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
@@ -96,27 +102,33 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
 deps = ["Mmap"]
 uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 
+[[DensityInterface]]
+deps = ["InverseFunctions", "Test"]
+git-tree-sha1 = "80c3e8639e3353e5d2912fb3a1916b8455e2494b"
+uuid = "b429d917-457f-4dbc-8f4c-0cc954292b1d"
+version = "0.4.0"
+
 [[Distances]]
-deps = ["LinearAlgebra", "Statistics", "StatsAPI"]
-git-tree-sha1 = "abe4ad222b26af3337262b8afb28fab8d215e9f8"
+deps = ["LinearAlgebra", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "3258d0659f812acde79e8a74b11f17ac06d0ca04"
 uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
-version = "0.10.3"
+version = "0.10.7"
 
 [[Distributed]]
 deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[Distributions]]
-deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"]
-git-tree-sha1 = "3889f646423ce91dd1055a76317e9a1d3a23fff1"
+deps = ["ChainRulesCore", "DensityInterface", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns", "Test"]
+git-tree-sha1 = "6a8dc9f82e5ce28279b6e3e2cea9421154f5bd0d"
 uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
-version = "0.25.11"
+version = "0.25.37"
 
 [[DocStringExtensions]]
 deps = ["LibGit2"]
-git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
+git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b"
 uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
-version = "0.8.5"
+version = "0.8.6"
 
 [[Downloads]]
 deps = ["ArgTools", "LibCURL", "NetworkOptions"]
@@ -124,9 +136,9 @@ uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 
 [[EarCut_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "92d8f9f208637e8d2d28c664051a00569c01493d"
+git-tree-sha1 = "3f3a2501fa7236e9b911e0f7a588c657e822bb6d"
 uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5"
-version = "2.1.5+1"
+version = "2.2.3+0"
 
 [[Expat_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -141,16 +153,16 @@ uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a"
 version = "0.4.1"
 
 [[FFMPEG_jll]]
-deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "LibVPX_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
-git-tree-sha1 = "3cc57ad0a213808473eafef4845a74766242e05f"
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
+git-tree-sha1 = "d8a578692e3077ac998b50c0217dfd67f21d1e5f"
 uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
-version = "4.3.1+4"
+version = "4.4.0+0"
 
 [[FillArrays]]
 deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"]
-git-tree-sha1 = "8c8eac2af06ce35973c3eadb4ab3243076a408e7"
+git-tree-sha1 = "8756f9935b7ccc9064c6eef0bff0ad643df733a3"
 uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "0.12.1"
+version = "0.12.7"
 
 [[FixedPointNumbers]]
 deps = ["Statistics"]
@@ -160,9 +172,9 @@ version = "0.8.4"
 
 [[Fontconfig_jll]]
 deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "35895cf184ceaab11fd778b4590144034a167a2f"
+git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03"
 uuid = "a3f928ae-7b40-5064-980b-68af3947d34b"
-version = "2.13.1+14"
+version = "2.13.93+0"
 
 [[Formatting]]
 deps = ["Printf"]
@@ -172,9 +184,9 @@ version = "0.4.2"
 
 [[FreeType2_jll]]
 deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "cbd58c9deb1d304f5a245a0b7eb841a2560cfec6"
+git-tree-sha1 = "87eb71354d8ec1a96d4a7636bd57a7347dde3ef9"
 uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7"
-version = "2.10.1+5"
+version = "2.10.4+0"
 
 [[FriBidi_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -183,28 +195,27 @@ uuid = "559328eb-81f9-559d-9380-de523a88c83c"
 version = "1.0.10+0"
 
 [[Functors]]
-deps = ["MacroTools"]
-git-tree-sha1 = "4cd9e70bf8fce05114598b663ad79dfe9ae432b3"
+git-tree-sha1 = "e4768c3b7f597d5a352afa09874d16e3c3f6ead2"
 uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
-version = "0.2.3"
+version = "0.2.7"
 
 [[GLFW_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"]
-git-tree-sha1 = "dba1e8614e98949abfa60480b13653813d8f0157"
+git-tree-sha1 = "0c603255764a1fa0b61752d2bec14cfbd18f7fe8"
 uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89"
-version = "3.3.5+0"
+version = "3.3.5+1"
 
 [[GR]]
 deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"]
-git-tree-sha1 = "182da592436e287758ded5be6e32c406de3a2e47"
+git-tree-sha1 = "b9a93bcdf34618031891ee56aad94cfff0843753"
 uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
-version = "0.58.1"
+version = "0.63.0"
 
 [[GR_jll]]
 deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"]
-git-tree-sha1 = "d59e8320c2747553788e4fc42231489cc602fa50"
+git-tree-sha1 = "f97acd98255568c3c9b416c5a3cf246c1315771b"
 uuid = "d2c73de3-f751-5644-a686-071e5b155ba9"
-version = "0.58.1+0"
+version = "0.63.0+0"
 
 [[GeometryBasics]]
 deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"]
@@ -220,9 +231,15 @@ version = "0.21.0+0"
 
 [[Glib_jll]]
 deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "7bf67e9a481712b3dbe9cb3dac852dc4b1162e02"
+git-tree-sha1 = "a32d672ac2c967f3deb8a81d828afc739c838a06"
 uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
-version = "2.68.3+0"
+version = "2.68.3+2"
+
+[[Graphite2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011"
+uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472"
+version = "1.3.14+0"
 
 [[Grisu]]
 git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2"
@@ -231,9 +248,15 @@ version = "1.0.2"
 
 [[HTTP]]
 deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
-git-tree-sha1 = "44e3b40da000eab4ccb1aecdc4801c040026aeb5"
+git-tree-sha1 = "0fa77022fe4b511826b39c894c90daf5fce3334a"
 uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "0.9.13"
+version = "0.9.17"
+
+[[HarfBuzz_jll]]
+deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"]
+git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3"
+uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566"
+version = "2.8.1+1"
 
 [[IOCapture]]
 deps = ["Logging", "Random"]
@@ -251,15 +274,21 @@ version = "0.5.0"
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
+[[InverseFunctions]]
+deps = ["Test"]
+git-tree-sha1 = "a7254c0acd8e62f1ac75ad24d5db43f5f19f3c65"
+uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
+version = "0.1.2"
+
 [[IrrationalConstants]]
-git-tree-sha1 = "f76424439413893a832026ca355fe273e93bce94"
+git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151"
 uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
-version = "0.1.0"
+version = "0.1.1"
 
 [[IterTools]]
-git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18"
+git-tree-sha1 = "fa6287a4469f5e048d763df38279ee729fbd44e5"
 uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
-version = "1.3.0"
+version = "1.4.0"
 
 [[IteratorInterfaceExtensions]]
 git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
@@ -288,7 +317,7 @@ version = "2.1.0+0"
 deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Random", "Requires", "SpecialFunctions", "StatsBase", "TensorCore", "Test", "ZygoteRules"]
 path = "../.."
 uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
-version = "0.10.12"
+version = "0.10.27"
 
 [[LAME_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -304,9 +333,9 @@ version = "0.6.0"
 
 [[LIBSVM]]
 deps = ["LIBLINEAR", "LinearAlgebra", "ScikitLearnBase", "SparseArrays", "libsvm_jll"]
-git-tree-sha1 = "729ea2db931587c983d0ef6691b62de5005c5570"
+git-tree-sha1 = "a5e607649aeb9ae3bbde19dc629faaa3b3d8955d"
 uuid = "b1bec4e5-fd48-53fe-b0cb-9723c09d164b"
-version = "0.7.0"
+version = "0.8.0"
 
 [[LZO_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -315,15 +344,15 @@ uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
 version = "2.10.1+0"
 
 [[LaTeXStrings]]
-git-tree-sha1 = "c7f1c695e06c01b95a67f0cd1d34994f3e7db104"
+git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996"
 uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
-version = "1.2.1"
+version = "1.3.0"
 
 [[Latexify]]
 deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"]
-git-tree-sha1 = "a4b12a1bd2ebade87891ab7e36fdbce582301a92"
+git-tree-sha1 = "a8f4f279b6fa3c3c4f1adadd78a621b13a506bce"
 uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
-version = "0.15.6"
+version = "0.15.9"
 
 [[LibCURL]]
 deps = ["LibCURL_jll", "MozillaCACerts_jll"]
@@ -341,20 +370,14 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
 uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
 
-[[LibVPX_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "12ee7e23fa4d18361e7c2cde8f8337d4c3101bc7"
-uuid = "dd192d2f-8180-539f-9fb4-cc70b1dcf69a"
-version = "1.10.0+0"
-
 [[Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [[Libffi_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "761a393aeccd6aa92ec3515e428c26bf99575b3b"
+git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290"
 uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490"
-version = "3.2.2+0"
+version = "3.2.2+1"
 
 [[Libgcrypt_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"]
@@ -404,24 +427,24 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [[Literate]]
 deps = ["Base64", "IOCapture", "JSON", "REPL"]
-git-tree-sha1 = "04a71882b041c0d896e4b379bb333a2e2d04054e"
+git-tree-sha1 = "d3493acfb9e6aa0cff46b09773fc2342327b0feb"
 uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
-version = "2.9.1"
+version = "2.9.4"
 
 [[LogExpFunctions]]
-deps = ["DocStringExtensions", "LinearAlgebra"]
-git-tree-sha1 = "7bd5f6565d80b6bf753738d2bc40a5dfea072070"
+deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"]
+git-tree-sha1 = "e5718a00af0ab9756305a0392832c8952c7426c1"
 uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
-version = "0.2.5"
+version = "0.3.6"
 
 [[Logging]]
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
 [[MacroTools]]
 deps = ["Markdown", "Random"]
-git-tree-sha1 = "0fb723cd8c45858c22169b2e42269e53271a6df7"
+git-tree-sha1 = "3d3e902b31198a27340d0bf00d6ac452866021cf"
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.5.7"
+version = "0.5.9"
 
 [[Markdown]]
 deps = ["Base64"]
@@ -444,9 +467,9 @@ version = "0.3.1"
 
 [[Missings]]
 deps = ["DataAPI"]
-git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7"
+git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f"
 uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
-version = "1.0.0"
+version = "1.0.2"
 
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -455,24 +478,28 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
 
 [[NaNMath]]
-git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb"
+git-tree-sha1 = "f755f36b19a5116bb580de457cda0c140153f283"
 uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
-version = "0.3.5"
+version = "0.3.6"
 
 [[NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 
 [[Ogg_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f"
+git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f"
 uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051"
-version = "1.3.5+0"
+version = "1.3.5+1"
+
+[[OpenLibm_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
 
 [[OpenSSL_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a"
+git-tree-sha1 = "648107615c15d4e09f7eca16307bc821c1f718d8"
 uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
-version = "1.1.10+0"
+version = "1.1.13+0"
 
 [[OpenSpecFun_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
@@ -499,15 +526,15 @@ version = "8.44.0+0"
 
 [[PDMats]]
 deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
-git-tree-sha1 = "4dd403333bcf0909341cfe57ec115152f937d7d8"
+git-tree-sha1 = "ee26b350276c51697c9c2d88a072b339f9f03d73"
 uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
-version = "0.11.1"
+version = "0.11.5"
 
 [[Parsers]]
 deps = ["Dates"]
-git-tree-sha1 = "477bf42b4d1496b454c10cce46645bb5b8a0cf2c"
+git-tree-sha1 = "d7fa6237da8004be601e19bd6666083056649918"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "2.0.2"
+version = "2.1.3"
 
 [[Pixman_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -527,21 +554,21 @@ version = "2.0.1"
 
 [[PlotUtils]]
 deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"]
-git-tree-sha1 = "501c20a63a34ac1d015d5304da0e645f42d91c9f"
+git-tree-sha1 = "68604313ed59f0408313228ba09e79252e4b2da8"
 uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043"
-version = "1.0.11"
+version = "1.1.2"
 
 [[Plots]]
-deps = ["Base64", "Contour", "Dates", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"]
-git-tree-sha1 = "e39bea10478c6aff5495ab522517fae5134b40e3"
+deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun", "Unzip"]
+git-tree-sha1 = "68e602f447344154f3b80f7d14bfb459a0f4dadf"
 uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
-version = "1.20.0"
+version = "1.25.5"
 
 [[Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
+git-tree-sha1 = "2cf929d64681236a2e074ffafb8d568733d2e6af"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.2.2"
+version = "1.2.3"
 
 [[Printf]]
 deps = ["Unicode"]
@@ -555,9 +582,9 @@ version = "5.15.3+0"
 
 [[QuadGK]]
 deps = ["DataStructures", "LinearAlgebra"]
-git-tree-sha1 = "12fbe86da16df6679be7521dfb39fbc861e1dc7b"
+git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39"
 uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
-version = "2.4.1"
+version = "2.4.2"
 
 [[REPL]]
 deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
@@ -568,26 +595,26 @@ deps = ["Serialization"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
 [[RecipesBase]]
-git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae"
+git-tree-sha1 = "6bf3f380ff52ce0832ddd3a2a7b9538ed1bcca7d"
 uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
-version = "1.1.1"
+version = "1.2.1"
 
 [[RecipesPipeline]]
 deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"]
-git-tree-sha1 = "2a7a2469ed5d94a98dea0e85c46fa653d76be0cd"
+git-tree-sha1 = "7ad0dfa8d03b7bcf8c597f59f5292801730c55b8"
 uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c"
-version = "0.3.4"
+version = "0.4.1"
 
 [[Reexport]]
-git-tree-sha1 = "5f6c21241f0f655da3952fd60aa18477cf96c220"
+git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
 uuid = "189a3867-3050-52da-a836-e630ba90ab69"
-version = "1.1.0"
+version = "1.2.2"
 
 [[Requires]]
 deps = ["UUIDs"]
-git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621"
+git-tree-sha1 = "8f82019e525f4d5c669692772a6f4b0a58b06a6a"
 uuid = "ae029012-a4dd-5104-9daa-d747884805df"
-version = "1.1.3"
+version = "1.2.0"
 
 [[Rmath]]
 deps = ["Random", "Rmath_jll"]
@@ -643,43 +670,43 @@ deps = ["LinearAlgebra", "Random"]
 uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [[SpecialFunctions]]
-deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"]
-git-tree-sha1 = "a322a9493e49c5f3a10b50df3aedaf1cdb3244b7"
+deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
+git-tree-sha1 = "e08890d19787ec25029113e88c34ec20cac1c91e"
 uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
-version = "1.6.1"
+version = "2.0.0"
 
 [[StaticArrays]]
 deps = ["LinearAlgebra", "Random", "Statistics"]
-git-tree-sha1 = "3240808c6d463ac46f1c1cd7638375cd22abbccb"
+git-tree-sha1 = "88a559da57529581472320892576a486fa2377b9"
 uuid = "90137ffa-7385-5640-81b9-e52037218182"
-version = "1.2.12"
+version = "1.3.1"
 
 [[Statistics]]
 deps = ["LinearAlgebra", "SparseArrays"]
 uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [[StatsAPI]]
-git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
+git-tree-sha1 = "d88665adc9bcf45903013af0982e2fd05ae3d0a6"
 uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
-version = "1.0.0"
+version = "1.2.0"
 
 [[StatsBase]]
-deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
-git-tree-sha1 = "fed1ec1e65749c4d96fc20dd13bea72b55457e62"
+deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "51383f2d367eb3b444c961d485c565e4c0cf4ba0"
 uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
-version = "0.33.9"
+version = "0.33.14"
 
 [[StatsFuns]]
-deps = ["LogExpFunctions", "Rmath", "SpecialFunctions"]
-git-tree-sha1 = "30cd8c360c54081f806b1ee14d2eecbef3c04c49"
+deps = ["ChainRulesCore", "InverseFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"]
+git-tree-sha1 = "bedb3e17cc1d94ce0e6e66d3afa47157978ba404"
 uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
-version = "0.9.8"
+version = "0.9.14"
 
 [[StructArrays]]
 deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"]
-git-tree-sha1 = "000e168f5cc9aded17b6999a560b7c11dda69095"
+git-tree-sha1 = "2ce41e0d042c60ecd131e9fb7154a3bfadbf50d3"
 uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
-version = "0.6.0"
+version = "0.6.3"
 
 [[SuiteSparse]]
 deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
@@ -697,9 +724,9 @@ version = "1.0.1"
 
 [[Tables]]
 deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
-git-tree-sha1 = "d0c690d37c73aeb5ca063056283fde5585a41710"
+git-tree-sha1 = "bb1064c9a84c52e277f1096cf41434b675cd368b"
 uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
-version = "1.5.0"
+version = "1.6.1"
 
 [[Tar]]
 deps = ["ArgTools", "SHA"]
@@ -727,6 +754,17 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 [[Unicode]]
 uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
+[[UnicodeFun]]
+deps = ["REPL"]
+git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf"
+uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1"
+version = "0.4.1"
+
+[[Unzip]]
+git-tree-sha1 = "34db80951901073501137bdbc3d5a8e7bbd06670"
+uuid = "41fe7b60-77ed-43a1-b4f0-825fd5a5650d"
+version = "0.1.2"
+
 [[Wayland_jll]]
 deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"]
 git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23"
@@ -734,10 +772,10 @@ uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89"
 version = "1.19.0+0"
 
 [[Wayland_protocols_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll"]
-git-tree-sha1 = "2839f1c1296940218e35df0bbb220f2a79686670"
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "66d72dc6fcc86352f01676e8f0f698562e60510f"
 uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91"
-version = "1.18.0+4"
+version = "1.23.0+0"
 
 [[XML2_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
@@ -889,21 +927,21 @@ version = "1.5.0+0"
 
 [[ZygoteRules]]
 deps = ["MacroTools"]
-git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7"
+git-tree-sha1 = "8c1a8e4dfacb1fd631745552c8db35d0deb09ea0"
 uuid = "700de1a5-db45-46bc-99cf-38207098b444"
-version = "0.2.1"
+version = "0.2.2"
 
 [[libass_jll]]
-deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "acc685bcf777b2202a904cdcb49ad34c2fa1880c"
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47"
 uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
-version = "0.14.0+4"
+version = "0.15.1+0"
 
 [[libfdk_aac_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "7a5780a0d9c6864184b3a2eeeb833a0c871f00ab"
+git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55"
 uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280"
-version = "0.1.6+4"
+version = "2.0.2+0"
 
 [[liblinear_jll]]
 deps = ["Libdl", "Pkg"]
@@ -939,15 +977,15 @@ uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 
 [[x264_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "d713c1ce4deac133e3334ee12f4adff07f81778f"
+git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2"
 uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a"
-version = "2020.7.14+2"
+version = "2021.5.5+0"
 
 [[x265_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "487da2f8f2f0c8ee0e83f39d13037d6bbf0a45ab"
+git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9"
 uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76"
-version = "3.0.0+3"
+version = "3.5.0+0"
 
 [[xkbcommon_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"]
diff --git a/examples/support-vector-machine/Project.toml b/examples/support-vector-machine/Project.toml
index 43d3ca7dc..9e527b4c4 100644
--- a/examples/support-vector-machine/Project.toml
+++ b/examples/support-vector-machine/Project.toml
@@ -9,7 +9,7 @@ Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 [compat]
 Distributions = "0.25"
 KernelFunctions = "0.10"
-LIBSVM = "0.7"
+LIBSVM = "0.7, 0.8"
 Literate = "2"
 Plots = "1"
 julia = "1.3"
diff --git a/examples/support-vector-machine/script.jl b/examples/support-vector-machine/script.jl
index f2824d4d4..ce68c15f2 100644
--- a/examples/support-vector-machine/script.jl
+++ b/examples/support-vector-machine/script.jl
@@ -1,5 +1,8 @@
 # # Support Vector Machine
 #
+# In this notebook we show how you can use KernelFunctions.jl to generate
+# kernel matrices for classification with a support vector machine, as
+# implemented by [LIBSVM](https://github.com/JuliaML/LIBSVM.jl).
 
 using Distributions
 using KernelFunctions
@@ -8,39 +11,52 @@ using LinearAlgebra
 using Plots
 using Random
 
-## Set plotting theme
-theme(:wong)
-
 ## Set seed
 Random.seed!(1234);
 
-# Number of samples:
-N = 100;
+# ## Generate half-moon dataset
 
-# Select randomly between two classes:
-y_train = rand([-1, 1], N);
+# Number of samples per class:
+n1 = n2 = 50;
 
-# Random attributes for both classes:
-X = Matrix{Float64}(undef, 2, N)
-rand!(MvNormal(randn(2), I), view(X, :, y_train .== 1))
-rand!(MvNormal(randn(2), I), view(X, :, y_train .== -1));
-x_train = ColVecs(X);
+# We generate data based on SciKit-Learn's sklearn.datasets.make_moons function:
 
-# Create a 2D grid:
-test_range = range(floor(Int, minimum(X)), ceil(Int, maximum(X)); length=100)
-x_test = ColVecs(mapreduce(collect, hcat, Iterators.product(test_range, test_range)));
+angle1 = range(0, π; length=n1)
+angle2 = range(0, π; length=n2)
+X1 = [cos.(angle1) sin.(angle1)] .+ 0.1 .* randn.()
+X2 = [1 .- cos.(angle2) 1 .- sin.(angle2) .- 0.5] .+ 0.1 .* randn.()
+X = [X1; X2]
+x_train = RowVecs(X)
+y_train = vcat(fill(-1, n1), fill(1, n2));
 
-# Create kernel function:
-k = SqExponentialKernel() ∘ ScaleTransform(2.0)
+# ## Training
+#
+# We create a kernel function:
+k = SqExponentialKernel() ∘ ScaleTransform(1.5)
 
-# [LIBSVM](https://github.com/JuliaML/LIBSVM.jl) can make use of a pre-computed kernel matrix.
-# KernelFunctions.jl can be used to produce that.
-# Precomputed matrix for training (corresponds to linear kernel)
+# LIBSVM can make use of a pre-computed kernel matrix.
+# KernelFunctions.jl can be used to produce that using `kernelmatrix`:
 model = svmtrain(kernelmatrix(k, x_train), y_train; kernel=LIBSVM.Kernel.Precomputed)
 
-# Precomputed matrix for prediction
-y_pr, _ = svmpredict(model, kernelmatrix(k, x_train, x_test));
+# ## Prediction
+#
+# For evaluation, we create a 100×100 2D grid based on the extent of the training data:
+test_range = range(floor(Int, minimum(X)), ceil(Int, maximum(X)); length=100)
+x_test = ColVecs(mapreduce(collect, hcat, Iterators.product(test_range, test_range)));
 
-# Compute prediction on a grid:
-contourf(test_range, test_range, y_pr)
-scatter!(X[1, :], X[2, :]; color=y_train, lab="data", widen=false)
+# Again, we pass the result of KernelFunctions.jl's `kernelmatrix` to LIBSVM:
+y_pred, _ = svmpredict(model, kernelmatrix(k, x_train, x_test));
+
+# We can see that the kernelized, non-linear classification successfully separates the two classes in the training data:
+plot(; lim=extrema(test_range), aspect_ratio=1)
+contourf!(
+    test_range,
+    test_range,
+    y_pred;
+    levels=1,
+    color=cgrad(:redsblues),
+    alpha=0.7,
+    colorbar_title="prediction",
+)
+scatter!(X1[:, 1], X1[:, 2]; color=:red, label="training data: class –1")
+scatter!(X2[:, 1], X2[:, 2]; color=:blue, label="training data: class 1")
diff --git a/src/approximations/nystrom.jl b/src/approximations/nystrom.jl
index a5cb1c035..674d9c8a9 100644
--- a/src/approximations/nystrom.jl
+++ b/src/approximations/nystrom.jl
@@ -1,25 +1,29 @@
 # Following the algorithm by William and Seeger, 2001
 # Cs is equivalent to X_mm and C to X_mn
 
-function sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs)
+function sampleindex(X::AbstractVector, r::Real)
     0 < r <= 1 || throw(ArgumentError("Sample rate `r` must be in range (0,1]"))
-    n = size(X, obsdim)
+    n = length(X)
     m = ceil(Int, n * r)
     S = StatsBase.sample(1:n, m; replace=false, ordered=true)
     return S
 end
 
-function nystrom_sample(
-    k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Integer=defaultobs
-)
-    obsdim ∈ [1, 2] ||
-        throw(ArgumentError("`obsdim` should be 1 or 2 (see docs of kernelmatrix))"))
-    Xₘ = obsdim == 1 ? X[S, :] : X[:, S]
-    C = kernelmatrix(k, Xₘ, X; obsdim=obsdim)
+@deprecate sampleindex(X::AbstractMatrix, r::Real; obsdim::Integer=defaultobs) sampleindex(
+    vec_of_vecs(X; obsdim=obsdim), r
+) false
+
+function nystrom_sample(k::Kernel, X::AbstractVector, S::AbstractVector{<:Integer})
+    Xₘ = @view X[S]
+    C = kernelmatrix(k, Xₘ, X)
     Cs = C[:, S]
     return (C, Cs)
 end
 
+@deprecate nystrom_sample(
+    k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Integer=defaultobs
+) nystrom_sample(k, vec_of_vecs(X; obsdim=obsdim), S) false
+
 function nystrom_pinv!(Cs::Matrix{T}, tol::T=eps(T) * size(Cs, 1)) where {T<:Real}
     # Compute eigendecomposition of sampled component of K
     QΛQᵀ = LinearAlgebra.eigen!(LinearAlgebra.Symmetric(Cs))
@@ -63,38 +67,48 @@ function NystromFact(W::Matrix{<:Real}, C::Matrix{<:Real})
 end
 
 @doc raw"""
-    nystrom(k::Kernel, X::Matrix, S::Vector; obsdim::Int=defaultobs)
+    nystrom(k::Kernel, X::AbstractVector, S::AbstractVector{<:Integer})
 
-Computes a factorization of Nystrom approximation of the square kernel matrix of data
-matrix `X` with respect to kernel `k`. Returns a `NystromFact` struct which stores a
-Nystrom factorization satisfying:
+Compute a factorization of a Nystrom approximation of the square kernel matrix
+of data vector `X` with respect to kernel `k`, using indices `S`.
+Returns a `NystromFact` struct which stores a Nystrom factorization satisfying:
 ```math
 \mathbf{K} \approx \mathbf{C}^{\intercal}\mathbf{W}\mathbf{C}
 ```
 """
-function nystrom(k::Kernel, X::AbstractMatrix, S::Vector{<:Integer}; obsdim::Int=defaultobs)
-    C, Cs = nystrom_sample(k, X, S; obsdim=obsdim)
+function nystrom(k::Kernel, X::AbstractVector, S::AbstractVector{<:Integer})
+    C, Cs = nystrom_sample(k, X, S)
     W = nystrom_pinv!(Cs)
     return NystromFact(W, C)
 end
 
 @doc raw"""
-    nystrom(k::Kernel, X::Matrix, r::Real; obsdim::Int=defaultobs)
+    nystrom(k::Kernel, X::AbstractVector, r::Real)
 
-Computes a factorization of Nystrom approximation of the square kernel matrix of data
-matrix `X` with respect to kernel `k` using a sample ratio of `r`.
+Compute a factorization of a Nystrom approximation of the square kernel matrix
+of data vector `X` with respect to kernel `k` using a sample ratio of `r`.
 Returns a `NystromFact` struct which stores a Nystrom factorization satisfying:
 ```math
 \mathbf{K} \approx \mathbf{C}^{\intercal}\mathbf{W}\mathbf{C}
 ```
 """
+function nystrom(k::Kernel, X::AbstractVector, r::Real)
+    S = sampleindex(X, r)
+    return nystrom(k, X, S)
+end
+
+function nystrom(
+    k::Kernel, X::AbstractMatrix, S::AbstractVector{<:Integer}; obsdim::Int=defaultobs
+)
+    return nystrom(k, vec_of_vecs(X; obsdim=obsdim), S)
+end
+
 function nystrom(k::Kernel, X::AbstractMatrix, r::Real; obsdim::Int=defaultobs)
-    S = sampleindex(X, r; obsdim=obsdim)
-    return nystrom(k, X, S; obsdim=obsdim)
+    return nystrom(k, vec_of_vecs(X; obsdim=obsdim), r)
 end
 
 """
-    nystrom(CᵀWC::NystromFact)
+    kernelmatrix(CᵀWC::NystromFact)
 
 Compute the approximate kernel matrix based on the Nystrom factorization.
 """
diff --git a/src/basekernels/constant.jl b/src/basekernels/constant.jl
index 5996546f1..80087d758 100644
--- a/src/basekernels/constant.jl
+++ b/src/basekernels/constant.jl
@@ -73,8 +73,8 @@ end
 
 @functor ConstantKernel
 
-kappa(κ::ConstantKernel, x::Real) = first(κ.c) * one(x)
+kappa(κ::ConstantKernel, x::Real) = only(κ.c) * one(x)
 
 metric(::ConstantKernel) = Delta()
 
-Base.show(io::IO, κ::ConstantKernel) = print(io, "Constant Kernel (c = ", first(κ.c), ")")
+Base.show(io::IO, κ::ConstantKernel) = print(io, "Constant Kernel (c = ", only(κ.c), ")")
diff --git a/src/basekernels/exponential.jl b/src/basekernels/exponential.jl
index c7a788b8a..2061d40f9 100644
--- a/src/basekernels/exponential.jl
+++ b/src/basekernels/exponential.jl
@@ -137,7 +137,7 @@ end
 
 @functor GammaExponentialKernel
 
-kappa(κ::GammaExponentialKernel, d::Real) = exp(-d^first(κ.γ))
+kappa(κ::GammaExponentialKernel, d::Real) = exp(-d^only(κ.γ))
 
 metric(k::GammaExponentialKernel) = k.metric
 
@@ -145,6 +145,6 @@ iskroncompatible(::GammaExponentialKernel) = true
 
 function Base.show(io::IO, κ::GammaExponentialKernel)
     return print(
-        io, "Gamma Exponential Kernel (γ = ", first(κ.γ), ", metric = ", κ.metric, ")"
+        io, "Gamma Exponential Kernel (γ = ", only(κ.γ), ", metric = ", κ.metric, ")"
     )
 end
diff --git a/src/basekernels/fbm.jl b/src/basekernels/fbm.jl
index 213cb3c36..08c7b3695 100644
--- a/src/basekernels/fbm.jl
+++ b/src/basekernels/fbm.jl
@@ -28,16 +28,16 @@ function (κ::FBMKernel)(x::AbstractVector{<:Real}, y::AbstractVector{<:Real})
     modX = sum(abs2, x)
     modY = sum(abs2, y)
     modXY = sqeuclidean(x, y)
-    h = first(κ.h)
+    h = only(κ.h)
     return (modX^h + modY^h - modXY^h) / 2
 end
 
 function (κ::FBMKernel)(x::Real, y::Real)
-    return (abs2(x)^first(κ.h) + abs2(y)^first(κ.h) - abs2(x - y)^first(κ.h)) / 2
+    return (abs2(x)^only(κ.h) + abs2(y)^only(κ.h) - abs2(x - y)^only(κ.h)) / 2
 end
 
 function Base.show(io::IO, κ::FBMKernel)
-    return print(io, "Fractional Brownian Motion Kernel (h = ", first(κ.h), ")")
+    return print(io, "Fractional Brownian Motion Kernel (h = ", only(κ.h), ")")
 end
 
 _fbm(modX, modY, modXY, h) = (modX^h + modY^h - modXY^h) / 2
diff --git a/src/basekernels/matern.jl b/src/basekernels/matern.jl
index a3c20efd7..a1ae4dfcc 100644
--- a/src/basekernels/matern.jl
+++ b/src/basekernels/matern.jl
@@ -34,7 +34,7 @@ MaternKernel(; nu::Real=1.5, ν::Real=nu, metric=Euclidean()) = MaternKernel(ν,
 @functor MaternKernel
 
 @inline function kappa(κ::MaternKernel, d::Real)
-    result = _matern(first(κ.ν), d)
+    result = _matern(only(κ.ν), d)
     return ifelse(iszero(d), one(result), result)
 end
 
@@ -46,7 +46,7 @@ end
 metric(k::MaternKernel) = k.metric
 
 function Base.show(io::IO, κ::MaternKernel)
-    return print(io, "Matern Kernel (ν = ", first(κ.ν), ", metric = ", κ.metric, ")")
+    return print(io, "Matern Kernel (ν = ", only(κ.ν), ", metric = ", κ.metric, ")")
 end
 
 ## Matern12Kernel = ExponentialKernel aliased in exponential.jl
diff --git a/src/basekernels/polynomial.jl b/src/basekernels/polynomial.jl
index da686e2c9..e0c0bfcb0 100644
--- a/src/basekernels/polynomial.jl
+++ b/src/basekernels/polynomial.jl
@@ -26,11 +26,11 @@ LinearKernel(; c::Real=0.0) = LinearKernel(c)
 
 @functor LinearKernel
 
-kappa(κ::LinearKernel, xᵀy::Real) = xᵀy + first(κ.c)
+kappa(κ::LinearKernel, xᵀy::Real) = xᵀy + only(κ.c)
 
 metric(::LinearKernel) = DotProduct()
 
-Base.show(io::IO, κ::LinearKernel) = print(io, "Linear Kernel (c = ", first(κ.c), ")")
+Base.show(io::IO, κ::LinearKernel) = print(io, "Linear Kernel (c = ", only(κ.c), ")")
 
 """
     PolynomialKernel(; degree::Int=2, c::Real=0.0)
@@ -53,7 +53,7 @@ struct PolynomialKernel{Tc<:Real} <: SimpleKernel
 
     function PolynomialKernel{Tc}(degree::Int, c::Vector{Tc}) where {Tc}
         @check_args(PolynomialKernel, degree, degree >= one(degree), "degree ≥ 1")
-        @check_args(PolynomialKernel, c, first(c) >= zero(Tc), "c ≥ 0")
+        @check_args(PolynomialKernel, c, only(c) >= zero(Tc), "c ≥ 0")
         return new{Tc}(degree, c)
     end
 end
@@ -68,10 +68,10 @@ function Functors.functor(::Type{<:PolynomialKernel}, x)
     return (c=x.c,), reconstruct_polynomialkernel
 end
 
-kappa(κ::PolynomialKernel, xᵀy::Real) = (xᵀy + first(κ.c))^κ.degree
+kappa(κ::PolynomialKernel, xᵀy::Real) = (xᵀy + only(κ.c))^κ.degree
 
 metric(::PolynomialKernel) = DotProduct()
 
 function Base.show(io::IO, κ::PolynomialKernel)
-    return print(io, "Polynomial Kernel (c = ", first(κ.c), ", degree = ", κ.degree, ")")
+    return print(io, "Polynomial Kernel (c = ", only(κ.c), ", degree = ", κ.degree, ")")
 end
diff --git a/src/basekernels/rational.jl b/src/basekernels/rational.jl
index 8ed396b51..0300b4f0d 100644
--- a/src/basekernels/rational.jl
+++ b/src/basekernels/rational.jl
@@ -32,13 +32,13 @@ end
 @functor RationalKernel
 
 function kappa(κ::RationalKernel, d::Real)
-    return (one(d) + d / first(κ.α))^(-first(κ.α))
+    return (one(d) + d / only(κ.α))^(-only(κ.α))
 end
 
 metric(k::RationalKernel) = k.metric
 
 function Base.show(io::IO, κ::RationalKernel)
-    return print(io, "Rational Kernel (α = ", first(κ.α), ", metric = ", κ.metric, ")")
+    return print(io, "Rational Kernel (α = ", only(κ.α), ", metric = ", κ.metric, ")")
 end
 
 """
@@ -72,10 +72,10 @@ end
 @functor RationalQuadraticKernel
 
 function kappa(κ::RationalQuadraticKernel, d::Real)
-    return (one(d) + d^2 / (2 * first(κ.α)))^(-first(κ.α))
+    return (one(d) + d^2 / (2 * only(κ.α)))^(-only(κ.α))
 end
 function kappa(κ::RationalQuadraticKernel{<:Real,<:Euclidean}, d²::Real)
-    return (one(d²) + d² / (2 * first(κ.α)))^(-first(κ.α))
+    return (one(d²) + d² / (2 * only(κ.α)))^(-only(κ.α))
 end
 
 metric(k::RationalQuadraticKernel) = k.metric
@@ -83,7 +83,7 @@ metric(::RationalQuadraticKernel{<:Real,<:Euclidean}) = SqEuclidean()
 
 function Base.show(io::IO, κ::RationalQuadraticKernel)
     return print(
-        io, "Rational Quadratic Kernel (α = ", first(κ.α), ", metric = ", κ.metric, ")"
+        io, "Rational Quadratic Kernel (α = ", only(κ.α), ", metric = ", κ.metric, ")"
     )
 end
 
@@ -122,7 +122,7 @@ end
 @functor GammaRationalKernel
 
 function kappa(κ::GammaRationalKernel, d::Real)
-    return (one(d) + d^first(κ.γ) / first(κ.α))^(-first(κ.α))
+    return (one(d) + d^only(κ.γ) / only(κ.α))^(-only(κ.α))
 end
 
 metric(k::GammaRationalKernel) = k.metric
@@ -131,9 +131,9 @@ function Base.show(io::IO, κ::GammaRationalKernel)
     return print(
         io,
         "Gamma Rational Kernel (α = ",
-        first(κ.α),
+        only(κ.α),
         ", γ = ",
-        first(κ.γ),
+        only(κ.γ),
         ", metric = ",
         κ.metric,
         ")",
diff --git a/src/chainrules.jl b/src/chainrules.jl
index 4b69a827f..dde352f8c 100644
--- a/src/chainrules.jl
+++ b/src/chainrules.jl
@@ -118,7 +118,7 @@ function ChainRulesCore.rrule(s::Sinus, x::AbstractVector, y::AbstractVector)
     return val, evaluate_pullback
 end
 
-## Reverse Rulse SqMahalanobis
+## Reverse Rules SqMahalanobis
 
 function ChainRulesCore.rrule(
     dist::Distances.SqMahalanobis, a::AbstractVector, b::AbstractVector
diff --git a/src/distances/sinus.jl b/src/distances/sinus.jl
index 4bcf4bdf0..51d14c47d 100644
--- a/src/distances/sinus.jl
+++ b/src/distances/sinus.jl
@@ -2,10 +2,12 @@ struct Sinus{T} <: Distances.UnionSemiMetric
     r::Vector{T}
 end
 
+Sinus(r::Real) = Sinus([r])
+
 Distances.parameters(d::Sinus) = d.r
 @inline Distances.eval_op(::Sinus, a::Real, b::Real, p::Real) = abs2(sinpi(a - b) / p)
 @inline (dist::Sinus)(a::AbstractArray, b::AbstractArray) = Distances._evaluate(dist, a, b)
-@inline (dist::Sinus)(a::Number, b::Number) = abs2(sinpi(a - b) / first(dist.r))
+@inline (dist::Sinus)(a::Number, b::Number) = abs2(sinpi(a - b) / only(dist.r))
 
 Distances.result_type(::Sinus{T}, Ta::Type, Tb::Type) where {T} = promote_type(T, Ta, Tb)
 
diff --git a/src/kernels/gibbskernel.jl b/src/kernels/gibbskernel.jl
index 4ce416f7f..46e14995d 100644
--- a/src/kernels/gibbskernel.jl
+++ b/src/kernels/gibbskernel.jl
@@ -1,20 +1,24 @@
-"""
+@doc raw"""
     GibbsKernel(; lengthscale)
 
-# Definition
+Gibbs Kernel with lengthscale function `lengthscale`.
 
-The Gibbs kernel is non-stationary generalisation of the squared exponential
+The Gibbs kernel is a non-stationary generalisation of the squared exponential
 kernel. The lengthscale parameter ``l`` becomes a function of
 position ``l(x)``.
 
-For a constant function``l(x) = c``, one recovers the standard squared exponential kernel
-with lengthscale `c`.
+# Definition
 
+For inputs ``x, x'``, the Gibbs kernel with lengthscale function ``l(\cdot)``
+is defined as
 ```math
-k(x, y; l) = \\sqrt{ \\left(\\frac{2 l(x) l(y)}{l(x)^2 + l(y)^2} \\right) }
-\\quad \\rm{exp} \\left( - \\frac{(x - y)^2}{l(x)^2 + l(y)^2} \\right)
+k(x, x'; l) = \sqrt{\left(\frac{2 l(x) l(x')}{l(x)^2 + l(x')^2}\right)}
+\quad \exp{\left(-\frac{(x - x')^2}{l(x)^2 + l(x')^2}\right)}.
 ```
 
+For a constant function ``l \equiv c``, one recovers the [`SqExponentialKernel`](@ref)
+with lengthscale `c`.
+
 # References
 
 Mark N. Gibbs. "Bayesian Gaussian Processes for Regression and Classication." PhD thesis, 1997
diff --git a/src/kernels/scaledkernel.jl b/src/kernels/scaledkernel.jl
index 897bdda1a..f7c6b9eae 100644
--- a/src/kernels/scaledkernel.jl
+++ b/src/kernels/scaledkernel.jl
@@ -23,7 +23,7 @@ end
 
 @functor ScaledKernel
 
-(k::ScaledKernel)(x, y) = first(k.σ²) * k.kernel(x, y)
+(k::ScaledKernel)(x, y) = only(k.σ²) * k.kernel(x, y)
 
 function kernelmatrix(κ::ScaledKernel, x::AbstractVector, y::AbstractVector)
     return κ.σ² .* kernelmatrix(κ.kernel, x, y)
@@ -75,5 +75,5 @@ Base.show(io::IO, κ::ScaledKernel) = printshifted(io, κ, 0)
 
 function printshifted(io::IO, κ::ScaledKernel, shift::Int)
     printshifted(io, κ.kernel, shift)
-    return print(io, "\n" * ("\t"^(shift + 1)) * "- σ² = $(first(κ.σ²))")
+    return print(io, "\n" * ("\t"^(shift + 1)) * "- σ² = $(only(κ.σ²))")
 end
diff --git a/src/kernels/transformedkernel.jl b/src/kernels/transformedkernel.jl
index 94ae5c147..88e719ef1 100644
--- a/src/kernels/transformedkernel.jl
+++ b/src/kernels/transformedkernel.jl
@@ -28,10 +28,10 @@ function (k::TransformedKernel{<:SimpleKernel,<:ScaleTransform})(
 end
 
 function _scale(t::ScaleTransform, metric::Euclidean, x, y)
-    return first(t.s) * evaluate(metric, x, y)
+    return only(t.s) * evaluate(metric, x, y)
 end
 function _scale(t::ScaleTransform, metric::Union{SqEuclidean,DotProduct}, x, y)
-    return first(t.s)^2 * evaluate(metric, x, y)
+    return only(t.s)^2 * evaluate(metric, x, y)
 end
 _scale(t::ScaleTransform, metric, x, y) = evaluate(metric, t(x), t(y))
 
diff --git a/src/transform/ardtransform.jl b/src/transform/ardtransform.jl
index 4e71d0141..726d940ad 100644
--- a/src/transform/ardtransform.jl
+++ b/src/transform/ardtransform.jl
@@ -32,7 +32,7 @@ end
 
 dim(t::ARDTransform) = length(t.v)
 
-(t::ARDTransform)(x::Real) = first(t.v) * x
+(t::ARDTransform)(x::Real) = only(t.v) * x
 (t::ARDTransform)(x) = t.v .* x
 
 _map(t::ARDTransform, x::AbstractVector{<:Real}) = t.v' .* x
diff --git a/src/transform/periodic_transform.jl b/src/transform/periodic_transform.jl
index 3430a63a1..098262309 100644
--- a/src/transform/periodic_transform.jl
+++ b/src/transform/periodic_transform.jl
@@ -25,16 +25,16 @@ PeriodicTransform(f::Real) = PeriodicTransform([f])
 
 dim(t::PeriodicTransform) = 2
 
-(t::PeriodicTransform)(x::Real) = [sinpi(2 * first(t.f) * x), cospi(2 * first(t.f) * x)]
+(t::PeriodicTransform)(x::Real) = [sinpi(2 * only(t.f) * x), cospi(2 * only(t.f) * x)]
 
 function _map(t::PeriodicTransform, x::AbstractVector{<:Real})
-    return RowVecs(hcat(sinpi.((2 * first(t.f)) .* x), cospi.((2 * first(t.f)) .* x)))
+    return RowVecs(hcat(sinpi.((2 * only(t.f)) .* x), cospi.((2 * only(t.f)) .* x)))
 end
 
 function Base.isequal(t1::PeriodicTransform, t2::PeriodicTransform)
-    return isequal(first(t1.f), first(t2.f))
+    return isequal(only(t1.f), only(t2.f))
 end
 
 function Base.show(io::IO, t::PeriodicTransform)
-    return print(io, "Periodic Transform with frequency $(first(t.f))")
+    return print(io, "Periodic Transform with frequency $(only(t.f))")
 end
diff --git a/src/transform/scaletransform.jl b/src/transform/scaletransform.jl
index 4cd1c5443..164ed3b39 100644
--- a/src/transform/scaletransform.jl
+++ b/src/transform/scaletransform.jl
@@ -24,12 +24,12 @@ end
 
 set!(t::ScaleTransform, ρ::Real) = t.s .= [ρ]
 
-(t::ScaleTransform)(x) = first(t.s) * x
+(t::ScaleTransform)(x) = only(t.s) * x
 
-_map(t::ScaleTransform, x::AbstractVector{<:Real}) = first(t.s) .* x
-_map(t::ScaleTransform, x::ColVecs) = ColVecs(first(t.s) .* x.X)
-_map(t::ScaleTransform, x::RowVecs) = RowVecs(first(t.s) .* x.X)
+_map(t::ScaleTransform, x::AbstractVector{<:Real}) = only(t.s) .* x
+_map(t::ScaleTransform, x::ColVecs) = ColVecs(only(t.s) .* x.X)
+_map(t::ScaleTransform, x::RowVecs) = RowVecs(only(t.s) .* x.X)
 
-Base.isequal(t::ScaleTransform, t2::ScaleTransform) = isequal(first(t.s), first(t2.s))
+Base.isequal(t::ScaleTransform, t2::ScaleTransform) = isequal(only(t.s), only(t2.s))
 
-Base.show(io::IO, t::ScaleTransform) = print(io, "Scale Transform (s = ", first(t.s), ")")
+Base.show(io::IO, t::ScaleTransform) = print(io, "Scale Transform (s = ", only(t.s), ")")
diff --git a/src/utils.jl b/src/utils.jl
index 7eea4358c..75dd62110 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -178,7 +178,7 @@ function validate_inputs(x, y)
     if dim(x) != dim(y) # Passes by default if `dim` is not defined
         throw(
             DimensionMismatch(
-                "Dimensionality of x ($(dim(x))) not equality to that of y ($(dim(y)))"
+                "dimensionality of x ($(dim(x))) is not equal to that of y ($(dim(y)))"
             ),
         )
     end
diff --git a/test/Project.toml b/test/Project.toml
index ef3a56dc2..4c04f2c42 100644
--- a/test/Project.toml
+++ b/test/Project.toml
@@ -1,5 +1,6 @@
 [deps]
 AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9"
+Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000"
@@ -17,14 +18,15 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [compat]
 AxisArrays = "0.4.3"
-Distances = "= 0.10.0, = 0.10.1, = 0.10.2, = 0.10.3, = 0.10.4"
+Compat = "3"
+Distances = "0.10"
 Documenter = "0.25, 0.26, 0.27"
 FiniteDifferences = "0.10.8, 0.11, 0.12"
 ForwardDiff = "0.10"
 Functors = "0.2"
-Kronecker = "0.4"
+Kronecker = "0.4, 0.5"
 LogExpFunctions = "0.2, 0.3"
 PDMats = "0.9, 0.10, 0.11"
 ReverseDiff = "1.2"
-SpecialFunctions = "0.10, 1"
+SpecialFunctions = "0.10, 1, 2"
 Zygote = "0.4, 0.5, 0.6"
diff --git a/test/approximations/nystrom.jl b/test/approximations/nystrom.jl
index 5e9c6773d..14476340f 100644
--- a/test/approximations/nystrom.jl
+++ b/test/approximations/nystrom.jl
@@ -2,10 +2,16 @@
     dims = [10, 5]
     X = rand(dims...)
     k = SqExponentialKernel()
+    for obsdim in [1, 2]
+        Xv = vec_of_vecs(X; obsdim=obsdim)
+        @assert Xv isa Union{ColVecs,RowVecs}
+        @test kernelmatrix(k, Xv) ≈ kernelmatrix(nystrom(k, Xv, 1.0))
+        @test kernelmatrix(k, Xv) ≈ kernelmatrix(nystrom(k, Xv, collect(1:dims[obsdim])))
+    end
     for obsdim in [1, 2]
         @test kernelmatrix(k, X; obsdim=obsdim) ≈
-              kernelmatrix(nystrom(k, X, 1.0; obsdim=obsdim))
+            kernelmatrix(nystrom(k, X, 1.0; obsdim=obsdim))
         @test kernelmatrix(k, X; obsdim=obsdim) ≈
-              kernelmatrix(nystrom(k, X, collect(1:dims[obsdim]); obsdim=obsdim))
+            kernelmatrix(nystrom(k, X, collect(1:dims[obsdim]); obsdim=obsdim))
     end
 end
diff --git a/test/basekernels/constant.jl b/test/basekernels/constant.jl
index e18df9419..f626fb297 100644
--- a/test/basekernels/constant.jl
+++ b/test/basekernels/constant.jl
@@ -36,6 +36,6 @@
 
         # Standardised tests.
         TestUtils.test_interface(k, Float64)
-        test_ADs(c -> ConstantKernel(; c=first(c)), [c])
+        test_ADs(c -> ConstantKernel(; c=only(c)), [c])
     end
 end
diff --git a/test/basekernels/exponential.jl b/test/basekernels/exponential.jl
index a002bf29d..21586cadd 100644
--- a/test/basekernels/exponential.jl
+++ b/test/basekernels/exponential.jl
@@ -56,7 +56,7 @@
         @test metric(k2) isa WeightedEuclidean
         @test k2(v1, v2) ≈ k(v1, v2)
 
-        test_ADs(γ -> GammaExponentialKernel(; gamma=first(γ)), [1 + 0.5 * rand()])
+        test_ADs(γ -> GammaExponentialKernel(; gamma=only(γ)), [1 + 0.5 * rand()])
         test_params(k, ([γ],))
         TestUtils.test_interface(GammaExponentialKernel(; γ=1.36))
 
diff --git a/test/chainrules.jl b/test/chainrules.jl
index 51a545ba1..03c2c3b1f 100644
--- a/test/chainrules.jl
+++ b/test/chainrules.jl
@@ -21,7 +21,11 @@
     compare_gradient(:Zygote, [x, y]) do xy
         KernelFunctions.Sinus(r)(xy[1], xy[2])
     end
-    compare_gradient(:Zygote, [Q, x, y]) do xy
-        SqMahalanobis(xy[1])(xy[2], xy[3])
+    if VERSION < v"1.6"
+        @test_broken "Chain rule of SqMahalanobis is broken in Julia pre-1.6"
+    else
+        compare_gradient(:Zygote, [Q, x, y]) do Qxy
+            SqMahalanobis(Qxy[1])(Qxy[2], Qxy[3])
+        end
     end
 end
diff --git a/test/distances/sinus.jl b/test/distances/sinus.jl
index 91ba1b028..d903e765d 100644
--- a/test/distances/sinus.jl
+++ b/test/distances/sinus.jl
@@ -5,5 +5,6 @@
     d = KernelFunctions.Sinus(p)
     @test Distances.parameters(d) == p
     @test evaluate(d, A, B) == sum(abs2.(sinpi.(A - B) ./ p))
-    @test d(3.0, 2.0) == abs2(sinpi(3.0 - 2.0) / first(p))
+    d1 = KernelFunctions.Sinus(first(p))
+    @test d1(3.0, 2.0) == abs2(sinpi(3.0 - 2.0) / first(p))
 end
diff --git a/test/runtests.jl b/test/runtests.jl
index 63c8e8f89..c9bc932f3 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -14,6 +14,7 @@ using Zygote: Zygote
 using ForwardDiff: ForwardDiff
 using ReverseDiff: ReverseDiff
 using FiniteDifferences: FiniteDifferences
+using Compat: only
 
 using KernelFunctions: SimpleKernel, metric, kappa, ColVecs, RowVecs, TestUtils
 
diff --git a/test/test_utils.jl b/test/test_utils.jl
index 22fe9fb08..b8c349e37 100644
--- a/test/test_utils.jl
+++ b/test/test_utils.jl
@@ -45,7 +45,7 @@ const FDM = FiniteDifferences.central_fdm(5, 1)
 gradient(f, s::Symbol, args) = gradient(f, Val(s), args)
 
 function gradient(f, ::Val{:Zygote}, args)
-    g = first(Zygote.gradient(f, args))
+    g = only(Zygote.gradient(f, args))
     if isnothing(g)
         if args isa AbstractArray{<:Real}
             return zeros(size(args)) # To respect the same output as other ADs
@@ -66,7 +66,11 @@ function gradient(f, ::Val{:ReverseDiff}, args)
 end
 
 function gradient(f, ::Val{:FiniteDiff}, args)
-    return first(FiniteDifferences.grad(FDM, f, args))
+    return only(FiniteDifferences.grad(FDM, f, args))
+end
+
+function compare_gradient(f, ::Val{:FiniteDiff}, args)
+    @test_nowarn gradient(f, :FiniteDiff, args)
 end
 
 function compare_gradient(f, AD::Symbol, args)
@@ -88,7 +92,7 @@ testdiagfunction(k::MOKernel, A, B) = sum(kernelmatrix_diag(k, A, B))
 function test_ADs(
     kernelfunction, args=nothing; ADs=[:Zygote, :ForwardDiff, :ReverseDiff], dims=[3, 3]
 )
-    test_fd = test_FiniteDiff(kernelfunction, args, dims)
+    test_fd = test_AD(:FiniteDiff, kernelfunction, args, dims)
     if !test_fd.anynonpass
         for AD in ADs
             test_AD(AD, kernelfunction, args, dims)
@@ -114,70 +118,6 @@ function test_ADs(
     end
 end
 
-function test_FiniteDiff(kernelfunction, args=nothing, dims=[3, 3])
-    # Init arguments :
-    k = if args === nothing
-        kernelfunction()
-    else
-        kernelfunction(args)
-    end
-    rng = MersenneTwister(42)
-    @testset "FiniteDifferences" begin
-        if k isa SimpleKernel
-            for d in log.([eps(), rand(rng)])
-                @test_nowarn gradient(:FiniteDiff, [d]) do x
-                    kappa(k, exp(first(x)))
-                end
-            end
-        end
-        ## Testing Kernel Functions
-        x = rand(rng, dims[1])
-        y = rand(rng, dims[1])
-        @test_nowarn gradient(:FiniteDiff, x) do x
-            k(x, y)
-        end
-        if !(args === nothing)
-            @test_nowarn gradient(:FiniteDiff, args) do p
-                kernelfunction(p)(x, y)
-            end
-        end
-        ## Testing Kernel Matrices
-        A = rand(rng, dims...)
-        B = rand(rng, dims...)
-        for dim in 1:2
-            @test_nowarn gradient(:FiniteDiff, A) do a
-                testfunction(k, a, dim)
-            end
-            @test_nowarn gradient(:FiniteDiff, A) do a
-                testfunction(k, a, B, dim)
-            end
-            @test_nowarn gradient(:FiniteDiff, B) do b
-                testfunction(k, A, b, dim)
-            end
-            if !(args === nothing)
-                @test_nowarn gradient(:FiniteDiff, args) do p
-                    testfunction(kernelfunction(p), A, B, dim)
-                end
-            end
-
-            @test_nowarn gradient(:FiniteDiff, A) do a
-                testdiagfunction(k, a, dim)
-            end
-            @test_nowarn gradient(:FiniteDiff, A) do a
-                testdiagfunction(k, a, B, dim)
-            end
-            @test_nowarn gradient(:FiniteDiff, B) do b
-                testdiagfunction(k, A, b, dim)
-            end
-            if args !== nothing
-                @test_nowarn gradient(:FiniteDiff, args) do p
-                    testdiagfunction(kernelfunction(p), A, B, dim)
-                end
-            end
-        end
-    end
-end
-
 function test_FiniteDiff(k::MOKernel, dims=(in=3, out=2, obs=3))
     rng = MersenneTwister(42)
     @testset "FiniteDifferences" begin
@@ -224,68 +164,68 @@ end
 
 function test_AD(AD::Symbol, kernelfunction, args=nothing, dims=[3, 3])
     @testset "$(AD)" begin
-        # Test kappa function
         k = if args === nothing
             kernelfunction()
         else
             kernelfunction(args)
         end
         rng = MersenneTwister(42)
+
         if k isa SimpleKernel
-            for d in log.([eps(), rand(rng)])
-                compare_gradient(AD, [d]) do x
-                    kappa(k, exp(x[1]))
+            @testset "kappa function" begin
+                for d in log.([eps(), rand(rng)])
+                    compare_gradient(AD, [d]) do x
+                        kappa(k, exp(x[1]))
+                    end
                 end
             end
         end
-        # Testing kernel evaluations
-        x = rand(rng, dims[1])
-        y = rand(rng, dims[1])
-        compare_gradient(AD, x) do x
-            k(x, y)
-        end
-        compare_gradient(AD, y) do y
-            k(x, y)
-        end
-        if !(args === nothing)
-            compare_gradient(AD, args) do p
-                kernelfunction(p)(x, y)
-            end
-        end
-        # Testing kernel matrices
-        A = rand(rng, dims...)
-        B = rand(rng, dims...)
-        for dim in 1:2
-            compare_gradient(AD, A) do a
-                testfunction(k, a, dim)
-            end
-            compare_gradient(AD, A) do a
-                testfunction(k, a, B, dim)
+
+        @testset "kernel evaluations" begin
+            x = rand(rng, dims[1])
+            y = rand(rng, dims[1])
+            compare_gradient(AD, x) do x
+                k(x, y)
             end
-            compare_gradient(AD, B) do b
-                testfunction(k, A, b, dim)
+            compare_gradient(AD, y) do y
+                k(x, y)
             end
             if !(args === nothing)
-                compare_gradient(AD, args) do p
-                    testfunction(kernelfunction(p), A, dim)
+                @testset "hyperparameters" begin
+                    compare_gradient(AD, args) do p
+                        kernelfunction(p)(x, y)
+                    end
                 end
             end
+        end
 
-            compare_gradient(AD, A) do a
-                testdiagfunction(k, a, dim)
-            end
-            compare_gradient(AD, A) do a
-                testdiagfunction(k, a, B, dim)
-            end
-            compare_gradient(AD, B) do b
-                testdiagfunction(k, A, b, dim)
-            end
-            if args !== nothing
-                compare_gradient(AD, args) do p
-                    testdiagfunction(kernelfunction(p), A, dim)
+        @testset "kernel matrices" begin
+            A = rand(rng, dims...)
+            B = rand(rng, dims...)
+            @testset "$(_testfn)" for _testfn in (testfunction, testdiagfunction)
+                for dim in 1:2
+                    compare_gradient(AD, A) do a
+                        _testfn(k, a, dim)
+                    end
+                    compare_gradient(AD, A) do a
+                        _testfn(k, a, B, dim)
+                    end
+                    compare_gradient(AD, B) do b
+                        _testfn(k, A, b, dim)
+                    end
+                    if !(args === nothing)
+                        @testset "hyperparameters" begin
+                            compare_gradient(AD, args) do p
+                                _testfn(kernelfunction(p), A, dim)
+                            end
+                            compare_gradient(AD, args) do p
+                                _testfn(kernelfunction(p), A, B, dim)
+                            end
+                        end
+                    end
                 end
             end
-        end
+        end # kernel matrices
     end
 end
 
diff --git a/test/transform/chaintransform.jl b/test/transform/chaintransform.jl
index 6e3d8a44f..f8b19cffe 100644
--- a/test/transform/chaintransform.jl
+++ b/test/transform/chaintransform.jl
@@ -24,6 +24,8 @@
     @test repr(tp ∘ tf) == "Chain of 2 transforms:\n\t - $(tf) |> $(tp)"
     test_ADs(
         x -> SEKernel() ∘ (ScaleTransform(exp(x[1])) ∘ ARDTransform(exp.(x[2:4]))),
-        randn(rng, 4),
+        randn(rng, 4);
+        ADs=[:ForwardDiff, :ReverseDiff],  # explicitly pass ADs to exclude :Zygote
     )
+    @test_broken "test_AD of chain transform is currently broken in Zygote, see GitHub issue #263"
 end

From fea9646ea626d6887952ba81feee17cbd7b4ecd1 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 15:57:00 +0000
Subject: [PATCH 15/41] change title

---
 examples/train-kernel-parameters/script.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 7415a5c2f..384e74171 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -1,4 +1,4 @@
-# # Kernel Ridge Regression
+# # Train Kernel Parameters
 
 # In this example we show the two main methods to perform regression on a kernel from KernelFunctions.jl.
 

From b5c1513fe573cebdfde86bb03b26fe10de048713 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 15:59:37 +0000
Subject: [PATCH 16/41] Update manifest

---
 .../train-kernel-parameters/Manifest.toml     | 478 +++++++++++-------
 1 file changed, 282 insertions(+), 196 deletions(-)

diff --git a/examples/train-kernel-parameters/Manifest.toml b/examples/train-kernel-parameters/Manifest.toml
index 7a3bb3837..a8b46e154 100644
--- a/examples/train-kernel-parameters/Manifest.toml
+++ b/examples/train-kernel-parameters/Manifest.toml
@@ -1,10 +1,10 @@
 # This file is machine-generated - editing it directly is not advised
 
 [[AbstractFFTs]]
-deps = ["LinearAlgebra"]
-git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0"
+deps = ["ChainRulesCore", "LinearAlgebra"]
+git-tree-sha1 = "6f1d9bc1c08f9f4a8fa92e3ea3cb50153a1b40d4"
 uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
-version = "1.0.1"
+version = "1.1.0"
 
 [[AbstractTrees]]
 git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5"
@@ -13,30 +13,36 @@ version = "0.3.4"
 
 [[Adapt]]
 deps = ["LinearAlgebra"]
-git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7"
+git-tree-sha1 = "af92965fb30777147966f58acb05da51c5616b5f"
 uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
-version = "3.3.1"
+version = "3.3.3"
 
 [[ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
 
+[[ArrayInterface]]
+deps = ["Compat", "IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"]
+git-tree-sha1 = "ffc6588e17bcfcaa79dfa5b4f417025e755f83fc"
+uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
+version = "4.0.1"
+
 [[Artifacts]]
 uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 
 [[BFloat16s]]
-deps = ["LinearAlgebra", "Test"]
-git-tree-sha1 = "4af69e205efc343068dc8722b8dfec1ade89254a"
+deps = ["LinearAlgebra", "Printf", "Random", "Test"]
+git-tree-sha1 = "a598ecb0d717092b5539dbbe890c98bac842b072"
 uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b"
-version = "0.1.0"
+version = "0.2.0"
 
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[Bzip2_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "c3598e525718abcc440f69cc6d5f60dda0a1b61e"
+git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2"
 uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
-version = "1.0.6+5"
+version = "1.0.8+0"
 
 [[CEnum]]
 git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
@@ -44,28 +50,34 @@ uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
 version = "0.4.1"
 
 [[CUDA]]
-deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"]
-git-tree-sha1 = "82b2811f5888465d96b38c7bb12d8fb9c25838e1"
+deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"]
+git-tree-sha1 = "3eb49a30d6093dfb6cd60ea66ff681a7db87178a"
 uuid = "052768ef-5323-5732-b1bb-66c8b64840ba"
-version = "3.3.1"
+version = "3.7.1"
 
 [[Cairo_jll]]
 deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
-git-tree-sha1 = "e2f47f6d8337369411569fd45ae5753ca10394c6"
+git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2"
 uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
-version = "1.16.0+6"
+version = "1.16.1+1"
 
 [[ChainRules]]
-deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Statistics"]
-git-tree-sha1 = "e7e3200bd24b77bcc849e6616f7c2f0d45d70f5b"
+deps = ["ChainRulesCore", "Compat", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "Statistics"]
+git-tree-sha1 = "3ce62c97bd166b8c810889fba131531c61e41dc8"
 uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2"
-version = "0.8.17"
+version = "1.25.0"
 
 [[ChainRulesCore]]
 deps = ["Compat", "LinearAlgebra", "SparseArrays"]
-git-tree-sha1 = "be770c08881f7bb928dfd86d1ba83798f76cf62a"
+git-tree-sha1 = "54fc4400de6e5c3e27be6047da2ef6ba355511f8"
 uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-version = "0.10.9"
+version = "1.11.6"
+
+[[ChangesOfVariables]]
+deps = ["ChainRulesCore", "LinearAlgebra", "Test"]
+git-tree-sha1 = "bf98fa45a0a4cee295de98d4c1462be26345b9a1"
+uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
+version = "0.1.2"
 
 [[CodecZlib]]
 deps = ["TranscodingStreams", "Zlib_jll"]
@@ -74,10 +86,10 @@ uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
 version = "0.7.0"
 
 [[ColorSchemes]]
-deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random", "StaticArrays"]
-git-tree-sha1 = "c8fd01e4b736013bc61b704871d20503b33ea402"
+deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random"]
+git-tree-sha1 = "6b6f04f93710c71550ec7e16b650c1b9a612d0b6"
 uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
-version = "3.12.1"
+version = "3.16.0"
 
 [[ColorTypes]]
 deps = ["FixedPointNumbers", "Random"]
@@ -99,18 +111,18 @@ version = "0.3.0"
 
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
-git-tree-sha1 = "dc7dedc2c2aa9faf59a55c622760a25cbefbe941"
+git-tree-sha1 = "44c37b4636bc54afac5c574d2d02b625349d6582"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "3.31.0"
+version = "3.41.0"
 
 [[CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
 
 [[CompositionsBase]]
-git-tree-sha1 = "f3955eb38944e5dd0fabf8ca1e267d94941d34a5"
+git-tree-sha1 = "455419f7e328a1a2493cabc6428d79e951349769"
 uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
-version = "0.1.0"
+version = "0.1.1"
 
 [[Contour]]
 deps = ["StaticArrays"]
@@ -119,15 +131,15 @@ uuid = "d38c429a-6771-53c6-b99e-75d170b6e991"
 version = "0.5.7"
 
 [[DataAPI]]
-git-tree-sha1 = "ee400abb2298bd13bfc3df1c412ed228061a2385"
+git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8"
 uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.7.0"
+version = "1.9.0"
 
 [[DataStructures]]
 deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677"
+git-tree-sha1 = "3daef5523dd2e769dad2365274f760ff5f282c7d"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.18.9"
+version = "0.18.11"
 
 [[DataValueInterfaces]]
 git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
@@ -142,6 +154,12 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
 deps = ["Mmap"]
 uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 
+[[DensityInterface]]
+deps = ["InverseFunctions", "Test"]
+git-tree-sha1 = "80c3e8639e3353e5d2912fb3a1916b8455e2494b"
+uuid = "b429d917-457f-4dbc-8f4c-0cc954292b1d"
+version = "0.4.0"
+
 [[DiffResults]]
 deps = ["StaticArrays"]
 git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805"
@@ -149,32 +167,32 @@ uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
 version = "1.0.3"
 
 [[DiffRules]]
-deps = ["NaNMath", "Random", "SpecialFunctions"]
-git-tree-sha1 = "214c3fcac57755cfda163d91c58893a8723f93e9"
+deps = ["LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"]
+git-tree-sha1 = "9bc5dac3c8b6706b58ad5ce24cffd9861f07c94f"
 uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
-version = "1.0.2"
+version = "1.9.0"
 
 [[Distances]]
-deps = ["LinearAlgebra", "Statistics", "StatsAPI"]
-git-tree-sha1 = "abe4ad222b26af3337262b8afb28fab8d215e9f8"
+deps = ["LinearAlgebra", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "3258d0659f812acde79e8a74b11f17ac06d0ca04"
 uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
-version = "0.10.3"
+version = "0.10.7"
 
 [[Distributed]]
 deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[Distributions]]
-deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"]
-git-tree-sha1 = "2733323e5c02a9d7f48e7a3c4bc98d764fb704da"
+deps = ["ChainRulesCore", "DensityInterface", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns", "Test"]
+git-tree-sha1 = "5863b0b10512ed4add2b5ec07e335dc6121065a5"
 uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
-version = "0.25.6"
+version = "0.25.41"
 
 [[DocStringExtensions]]
 deps = ["LibGit2"]
-git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
+git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b"
 uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
-version = "0.8.5"
+version = "0.8.6"
 
 [[Downloads]]
 deps = ["ArgTools", "LibCURL", "NetworkOptions"]
@@ -182,9 +200,9 @@ uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 
 [[EarCut_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "92d8f9f208637e8d2d28c664051a00569c01493d"
+git-tree-sha1 = "3f3a2501fa7236e9b911e0f7a588c657e822bb6d"
 uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5"
-version = "2.1.5+1"
+version = "2.2.3+0"
 
 [[Expat_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -193,9 +211,9 @@ uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
 version = "2.2.10+0"
 
 [[ExprTools]]
-git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e"
+git-tree-sha1 = "56559bbef6ca5ea0c0818fa5c90320398a6fbf8d"
 uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
-version = "0.1.3"
+version = "0.1.8"
 
 [[FFMPEG]]
 deps = ["FFMPEG_jll"]
@@ -204,16 +222,16 @@ uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a"
 version = "0.4.1"
 
 [[FFMPEG_jll]]
-deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "LibVPX_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
-git-tree-sha1 = "3cc57ad0a213808473eafef4845a74766242e05f"
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
+git-tree-sha1 = "d8a578692e3077ac998b50c0217dfd67f21d1e5f"
 uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
-version = "4.3.1+4"
+version = "4.4.0+0"
 
 [[FillArrays]]
-deps = ["LinearAlgebra", "Random", "SparseArrays"]
-git-tree-sha1 = "a603e79b71bb3c1efdb58f0ee32286efe2d1a255"
+deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"]
+git-tree-sha1 = "8756f9935b7ccc9064c6eef0bff0ad643df733a3"
 uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "0.11.8"
+version = "0.12.7"
 
 [[FixedPointNumbers]]
 deps = ["Statistics"]
@@ -222,16 +240,16 @@ uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
 version = "0.8.4"
 
 [[Flux]]
-deps = ["AbstractTrees", "Adapt", "CUDA", "CodecZlib", "Colors", "DelimitedFiles", "Functors", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "NNlibCUDA", "Pkg", "Printf", "Random", "Reexport", "SHA", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"]
-git-tree-sha1 = "0b3c6d0ce57d3b793eabd346ccc8f605035ef079"
+deps = ["AbstractTrees", "Adapt", "ArrayInterface", "CUDA", "CodecZlib", "Colors", "DelimitedFiles", "Functors", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "NNlibCUDA", "Pkg", "Printf", "Random", "Reexport", "SHA", "SparseArrays", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"]
+git-tree-sha1 = "983271b47332fd3d9488d6f2d724570290971794"
 uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c"
-version = "0.12.4"
+version = "0.12.9"
 
 [[Fontconfig_jll]]
 deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "35895cf184ceaab11fd778b4590144034a167a2f"
+git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03"
 uuid = "a3f928ae-7b40-5064-980b-68af3947d34b"
-version = "2.13.1+14"
+version = "2.13.93+0"
 
 [[Formatting]]
 deps = ["Printf"]
@@ -240,16 +258,16 @@ uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
 version = "0.4.2"
 
 [[ForwardDiff]]
-deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
-git-tree-sha1 = "e2af66012e08966366a43251e1fd421522908be6"
+deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
+git-tree-sha1 = "1bd6fc0c344fc0cbee1f42f8d2e7ec8253dda2d2"
 uuid = "f6369f11-7733-5829-9624-2563aa707210"
-version = "0.10.18"
+version = "0.10.25"
 
 [[FreeType2_jll]]
 deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "cbd58c9deb1d304f5a245a0b7eb841a2560cfec6"
+git-tree-sha1 = "87eb71354d8ec1a96d4a7636bd57a7347dde3ef9"
 uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7"
-version = "2.10.1+5"
+version = "2.10.4+0"
 
 [[FriBidi_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -258,46 +276,45 @@ uuid = "559328eb-81f9-559d-9380-de523a88c83c"
 version = "1.0.10+0"
 
 [[Functors]]
-deps = ["MacroTools"]
-git-tree-sha1 = "a7bb2af991c43dcf5c3455d276dd83976799634f"
+git-tree-sha1 = "e4768c3b7f597d5a352afa09874d16e3c3f6ead2"
 uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
-version = "0.2.1"
+version = "0.2.7"
 
 [[GLFW_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"]
-git-tree-sha1 = "dba1e8614e98949abfa60480b13653813d8f0157"
+git-tree-sha1 = "0c603255764a1fa0b61752d2bec14cfbd18f7fe8"
 uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89"
-version = "3.3.5+0"
+version = "3.3.5+1"
 
 [[GPUArrays]]
-deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"]
-git-tree-sha1 = "ececbf05f8904c92814bdbd0aafd5540b0bf2e9a"
+deps = ["Adapt", "LLVM", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"]
+git-tree-sha1 = "cf91e6e9213b9190dc0511d6fff862a86652a94a"
 uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
-version = "7.0.1"
+version = "8.2.1"
 
 [[GPUCompiler]]
-deps = ["DataStructures", "ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "TimerOutputs", "UUIDs"]
-git-tree-sha1 = "222c6cdb888ec24795936d6829aa978691def60e"
+deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "TimerOutputs", "UUIDs"]
+git-tree-sha1 = "abd824e1f2ecd18d33811629c781441e94a24e81"
 uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
-version = "0.12.3"
+version = "0.13.11"
 
 [[GR]]
-deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"]
-git-tree-sha1 = "b83e3125048a9c3158cbb7ca423790c7b1b57bea"
+deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "RelocatableFolders", "Serialization", "Sockets", "Test", "UUIDs"]
+git-tree-sha1 = "4a740db447aae0fbeb3ee730de1afbb14ac798a1"
 uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
-version = "0.57.5"
+version = "0.63.1"
 
 [[GR_jll]]
 deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"]
-git-tree-sha1 = "e14907859a1d3aee73a019e7b3c98e9e7b8b5b3e"
+git-tree-sha1 = "aa22e1ee9e722f1da183eb33370df4c1aeb6c2cd"
 uuid = "d2c73de3-f751-5644-a686-071e5b155ba9"
-version = "0.57.3+0"
+version = "0.63.1+0"
 
 [[GeometryBasics]]
 deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"]
-git-tree-sha1 = "15ff9a14b9e1218958d3530cc288cf31465d9ae2"
+git-tree-sha1 = "58bcdf5ebc057b085e58d95c138725628dd7453c"
 uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
-version = "0.3.13"
+version = "0.4.1"
 
 [[Gettext_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"]
@@ -307,9 +324,15 @@ version = "0.21.0+0"
 
 [[Glib_jll]]
 deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "47ce50b742921377301e15005c96e979574e130b"
+git-tree-sha1 = "a32d672ac2c967f3deb8a81d828afc739c838a06"
 uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
-version = "2.68.1+0"
+version = "2.68.3+2"
+
+[[Graphite2_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011"
+uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472"
+version = "1.3.14+0"
 
 [[Grisu]]
 git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2"
@@ -318,9 +341,15 @@ version = "1.0.2"
 
 [[HTTP]]
 deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
-git-tree-sha1 = "99c6bc381e98c5331f34b9ce949aebea209bd0db"
+git-tree-sha1 = "0fa77022fe4b511826b39c894c90daf5fce3334a"
 uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "0.9.11"
+version = "0.9.17"
+
+[[HarfBuzz_jll]]
+deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"]
+git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3"
+uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566"
+version = "2.8.1+1"
 
 [[IOCapture]]
 deps = ["Logging", "Random"]
@@ -330,9 +359,14 @@ version = "0.2.2"
 
 [[IRTools]]
 deps = ["InteractiveUtils", "MacroTools", "Test"]
-git-tree-sha1 = "95215cd0076a150ef46ff7928892bc341864c73c"
+git-tree-sha1 = "006127162a51f0effbdfaab5ac0c83f8eb7ea8f3"
 uuid = "7869d1d1-7146-5819-86e3-90919afe41df"
-version = "0.4.3"
+version = "0.4.4"
+
+[[IfElse]]
+git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1"
+uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173"
+version = "0.1.1"
 
 [[IniFile]]
 deps = ["Test"]
@@ -344,10 +378,21 @@ version = "0.5.0"
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
+[[InverseFunctions]]
+deps = ["Test"]
+git-tree-sha1 = "a7254c0acd8e62f1ac75ad24d5db43f5f19f3c65"
+uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
+version = "0.1.2"
+
+[[IrrationalConstants]]
+git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151"
+uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
+version = "0.1.1"
+
 [[IterTools]]
-git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18"
+git-tree-sha1 = "fa6287a4469f5e048d763df38279ee729fbd44e5"
 uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
-version = "1.3.0"
+version = "1.4.0"
 
 [[IteratorInterfaceExtensions]]
 git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
@@ -356,15 +401,15 @@ version = "1.0.0"
 
 [[JLLWrappers]]
 deps = ["Preferences"]
-git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
+git-tree-sha1 = "22df5b96feef82434b07327e2d3c770a9b21e023"
 uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
-version = "1.3.0"
+version = "1.4.0"
 
 [[JSON]]
 deps = ["Dates", "Mmap", "Parsers", "Unicode"]
-git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
+git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-version = "0.21.1"
+version = "0.21.2"
 
 [[JpegTurbo_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -379,10 +424,10 @@ uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d"
 version = "0.8.4"
 
 [[KernelFunctions]]
-deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "LinearAlgebra", "Random", "Requires", "SpecialFunctions", "StatsBase", "StatsFuns", "TensorCore", "Test", "ZygoteRules"]
-git-tree-sha1 = "e8b5ba31b6d18695fd46bfcd8557682839023195"
+deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Random", "Requires", "SpecialFunctions", "StatsBase", "TensorCore", "Test", "ZygoteRules"]
+git-tree-sha1 = "1691afdf728eeb695b37005d8adc674c4e1612a7"
 uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
-version = "0.10.6"
+version = "0.10.28"
 
 [[LAME_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -391,10 +436,16 @@ uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d"
 version = "3.100.1+0"
 
 [[LLVM]]
-deps = ["CEnum", "Libdl", "Printf", "Unicode"]
-git-tree-sha1 = "f57ac3fd2045b50d3db081663837ac5b4096947e"
+deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"]
+git-tree-sha1 = "f8dcd7adfda0dddaf944e62476d823164cccc217"
 uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
-version = "3.9.0"
+version = "4.7.1"
+
+[[LLVMExtra_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "62115afed394c016c2d3096c5b85c407b48be96b"
+uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab"
+version = "0.0.13+1"
 
 [[LZO_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -403,15 +454,15 @@ uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
 version = "2.10.1+0"
 
 [[LaTeXStrings]]
-git-tree-sha1 = "c7f1c695e06c01b95a67f0cd1d34994f3e7db104"
+git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996"
 uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
-version = "1.2.1"
+version = "1.3.0"
 
 [[Latexify]]
 deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"]
-git-tree-sha1 = "a4b12a1bd2ebade87891ab7e36fdbce582301a92"
+git-tree-sha1 = "a8f4f279b6fa3c3c4f1adadd78a621b13a506bce"
 uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
-version = "0.15.6"
+version = "0.15.9"
 
 [[LazyArtifacts]]
 deps = ["Artifacts", "Pkg"]
@@ -433,20 +484,14 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
 uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
 
-[[LibVPX_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "12ee7e23fa4d18361e7c2cde8f8337d4c3101bc7"
-uuid = "dd192d2f-8180-539f-9fb4-cc70b1dcf69a"
-version = "1.10.0+0"
-
 [[Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [[Libffi_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "761a393aeccd6aa92ec3515e428c26bf99575b3b"
+git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290"
 uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490"
-version = "3.2.2+0"
+version = "3.2.2+1"
 
 [[Libgcrypt_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"]
@@ -491,29 +536,29 @@ uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700"
 version = "2.36.0+0"
 
 [[LinearAlgebra]]
-deps = ["Libdl"]
+deps = ["Libdl", "libblastrampoline_jll"]
 uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [[Literate]]
 deps = ["Base64", "IOCapture", "JSON", "REPL"]
-git-tree-sha1 = "501a1a74a0c825037860d36d87d703e987d39dbc"
+git-tree-sha1 = "a9e1c1ce0a604cf91b1698268cf05718fb4e25c8"
 uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
-version = "2.8.1"
+version = "2.11.0"
 
 [[LogExpFunctions]]
-deps = ["DocStringExtensions", "LinearAlgebra"]
-git-tree-sha1 = "1ba664552f1ef15325e68dc4c05c3ef8c2d5d885"
+deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"]
+git-tree-sha1 = "e5718a00af0ab9756305a0392832c8952c7426c1"
 uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
-version = "0.2.4"
+version = "0.3.6"
 
 [[Logging]]
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
 [[MacroTools]]
 deps = ["Markdown", "Random"]
-git-tree-sha1 = "6a8a2a625ab0dea913aba95c11370589e0239ff0"
+git-tree-sha1 = "3d3e902b31198a27340d0bf00d6ac452866021cf"
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.5.6"
+version = "0.5.9"
 
 [[Markdown]]
 deps = ["Base64"]
@@ -542,9 +587,9 @@ version = "0.5.0"
 
 [[Missings]]
 deps = ["DataAPI"]
-git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7"
+git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f"
 uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
-version = "1.0.0"
+version = "1.0.2"
 
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -554,35 +599,43 @@ uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
 
 [[NNlib]]
 deps = ["Adapt", "ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"]
-git-tree-sha1 = "7461639cef384a2ad058005b49e32b318d844343"
+git-tree-sha1 = "0b9f48ee4a793ae1d2c766093e9f45f405de6231"
 uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
-version = "0.7.22"
+version = "0.8.0"
 
 [[NNlibCUDA]]
 deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"]
-git-tree-sha1 = "c2f2f152340022ce44c6ac97e05c62a6de59c373"
+git-tree-sha1 = "adb43860f6371f42fb42d1d792c5d0d2f4cbc716"
 uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d"
-version = "0.1.4"
+version = "0.2.0"
 
 [[NaNMath]]
-git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb"
+git-tree-sha1 = "b086b7ea07f8e38cf122f5016af580881ac914fe"
 uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
-version = "0.3.5"
+version = "0.3.7"
 
 [[NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 
 [[Ogg_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f"
+git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f"
 uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051"
-version = "1.3.5+0"
+version = "1.3.5+1"
+
+[[OpenBLAS_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
+uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
+
+[[OpenLibm_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
 
 [[OpenSSL_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a"
+git-tree-sha1 = "648107615c15d4e09f7eca16307bc821c1f718d8"
 uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
-version = "1.1.10+0"
+version = "1.1.13+0"
 
 [[OpenSpecFun_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
@@ -609,15 +662,15 @@ version = "8.44.0+0"
 
 [[PDMats]]
 deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
-git-tree-sha1 = "4dd403333bcf0909341cfe57ec115152f937d7d8"
+git-tree-sha1 = "ee26b350276c51697c9c2d88a072b339f9f03d73"
 uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
-version = "0.11.1"
+version = "0.11.5"
 
 [[Parsers]]
 deps = ["Dates"]
-git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc"
+git-tree-sha1 = "0b5cfbb704034b5b4c1869e36634438a047df065"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "1.1.0"
+version = "2.2.1"
 
 [[Pixman_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -637,21 +690,21 @@ version = "2.0.1"
 
 [[PlotUtils]]
 deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"]
-git-tree-sha1 = "ae9a295ac761f64d8c2ec7f9f24d21eb4ffba34d"
+git-tree-sha1 = "6f1b25e8ea06279b5689263cc538f51331d7ca17"
 uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043"
-version = "1.0.10"
+version = "1.1.3"
 
 [[Plots]]
-deps = ["Base64", "Contour", "Dates", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"]
-git-tree-sha1 = "df601eed7c9637235a26b26f9f648deccd277178"
+deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun", "Unzip"]
+git-tree-sha1 = "7e4920a7d4323b8ffc3db184580598450bde8a8e"
 uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
-version = "1.16.7"
+version = "1.25.7"
 
 [[Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
+git-tree-sha1 = "2cf929d64681236a2e074ffafb8d568733d2e6af"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.2.2"
+version = "1.2.3"
 
 [[Printf]]
 deps = ["Unicode"]
@@ -669,9 +722,9 @@ version = "5.15.3+0"
 
 [[QuadGK]]
 deps = ["DataStructures", "LinearAlgebra"]
-git-tree-sha1 = "12fbe86da16df6679be7521dfb39fbc861e1dc7b"
+git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39"
 uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
-version = "2.4.1"
+version = "2.4.2"
 
 [[REPL]]
 deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
@@ -689,31 +742,43 @@ version = "1.4.2"
 
 [[RandomNumbers]]
 deps = ["Random", "Requires"]
-git-tree-sha1 = "441e6fc35597524ada7f85e13df1f4e10137d16f"
+git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111"
 uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143"
-version = "1.4.0"
+version = "1.5.3"
+
+[[RealDot]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9"
+uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9"
+version = "0.1.0"
 
 [[RecipesBase]]
-git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae"
+git-tree-sha1 = "6bf3f380ff52ce0832ddd3a2a7b9538ed1bcca7d"
 uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
-version = "1.1.1"
+version = "1.2.1"
 
 [[RecipesPipeline]]
 deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"]
-git-tree-sha1 = "9b8e57e3cca8828a1bc759840bfe48d64db9abfb"
+git-tree-sha1 = "37c1631cb3cc36a535105e6d5557864c82cd8c2b"
 uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c"
-version = "0.3.3"
+version = "0.5.0"
 
 [[Reexport]]
-git-tree-sha1 = "5f6c21241f0f655da3952fd60aa18477cf96c220"
+git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
 uuid = "189a3867-3050-52da-a836-e630ba90ab69"
-version = "1.1.0"
+version = "1.2.2"
+
+[[RelocatableFolders]]
+deps = ["SHA", "Scratch"]
+git-tree-sha1 = "cdbd3b1338c72ce29d9584fdbe9e9b70eeb5adca"
+uuid = "05181044-ff0b-4ac5-8273-598c1e38db00"
+version = "0.1.3"
 
 [[Requires]]
 deps = ["UUIDs"]
-git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621"
+git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
 uuid = "ae029012-a4dd-5104-9daa-d747884805df"
-version = "1.1.3"
+version = "1.3.0"
 
 [[Rmath]]
 deps = ["Random", "Rmath_jll"]
@@ -754,52 +819,58 @@ uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 
 [[SortingAlgorithms]]
 deps = ["DataStructures"]
-git-tree-sha1 = "2ec1962eba973f383239da22e75218565c390a96"
+git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508"
 uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
-version = "1.0.0"
+version = "1.0.1"
 
 [[SparseArrays]]
 deps = ["LinearAlgebra", "Random"]
 uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [[SpecialFunctions]]
-deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"]
-git-tree-sha1 = "a50550fa3164a8c46747e62063b4d774ac1bcf49"
+deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
+git-tree-sha1 = "e6bf188613555c78062842777b116905a9f9dd49"
 uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
-version = "1.5.1"
+version = "2.1.0"
+
+[[Static]]
+deps = ["IfElse"]
+git-tree-sha1 = "b4912cd034cdf968e06ca5f943bb54b17b97793a"
+uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3"
+version = "0.5.1"
 
 [[StaticArrays]]
 deps = ["LinearAlgebra", "Random", "Statistics"]
-git-tree-sha1 = "745914ebcd610da69f3cb6bf76cb7bb83dcb8c9a"
+git-tree-sha1 = "2884859916598f974858ff01df7dfc6c708dd895"
 uuid = "90137ffa-7385-5640-81b9-e52037218182"
-version = "1.2.4"
+version = "1.3.3"
 
 [[Statistics]]
 deps = ["LinearAlgebra", "SparseArrays"]
 uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [[StatsAPI]]
-git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
+git-tree-sha1 = "d88665adc9bcf45903013af0982e2fd05ae3d0a6"
 uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
-version = "1.0.0"
+version = "1.2.0"
 
 [[StatsBase]]
-deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
-git-tree-sha1 = "2f6792d523d7448bbe2fec99eca9218f06cc746d"
+deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "51383f2d367eb3b444c961d485c565e4c0cf4ba0"
 uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
-version = "0.33.8"
+version = "0.33.14"
 
 [[StatsFuns]]
-deps = ["LogExpFunctions", "Rmath", "SpecialFunctions"]
-git-tree-sha1 = "30cd8c360c54081f806b1ee14d2eecbef3c04c49"
+deps = ["ChainRulesCore", "InverseFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"]
+git-tree-sha1 = "f35e1879a71cca95f4826a14cdbf0b9e253ed918"
 uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
-version = "0.9.8"
+version = "0.9.15"
 
 [[StructArrays]]
 deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"]
-git-tree-sha1 = "000e168f5cc9aded17b6999a560b7c11dda69095"
+git-tree-sha1 = "d21f2c564b21a202f4677c0fba5b5ee431058544"
 uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
-version = "0.6.0"
+version = "0.6.4"
 
 [[SuiteSparse]]
 deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
@@ -817,9 +888,9 @@ version = "1.0.1"
 
 [[Tables]]
 deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
-git-tree-sha1 = "8ed4a3ea724dac32670b062be3ef1c1de6773ae8"
+git-tree-sha1 = "bb1064c9a84c52e277f1096cf41434b675cd368b"
 uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
-version = "1.4.4"
+version = "1.6.1"
 
 [[Tar]]
 deps = ["ArgTools", "SHA"]
@@ -837,15 +908,15 @@ uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [[TimerOutputs]]
 deps = ["ExprTools", "Printf"]
-git-tree-sha1 = "9f494bc54b4c31404a9eff449235836615929de1"
+git-tree-sha1 = "97e999be94a7147d0609d0b9fc9feca4bf24d76b"
 uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
-version = "0.5.10"
+version = "0.5.15"
 
 [[TranscodingStreams]]
 deps = ["Random", "Test"]
-git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
+git-tree-sha1 = "216b95ea110b5972db65aa90f88d8d89dcb8851c"
 uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
-version = "0.9.5"
+version = "0.9.6"
 
 [[URIs]]
 git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355"
@@ -859,6 +930,17 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 [[Unicode]]
 uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
+[[UnicodeFun]]
+deps = ["REPL"]
+git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf"
+uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1"
+version = "0.4.1"
+
+[[Unzip]]
+git-tree-sha1 = "34db80951901073501137bdbc3d5a8e7bbd06670"
+uuid = "41fe7b60-77ed-43a1-b4f0-825fd5a5650d"
+version = "0.1.2"
+
 [[Wayland_jll]]
 deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"]
 git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23"
@@ -866,10 +948,10 @@ uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89"
 version = "1.19.0+0"
 
 [[Wayland_protocols_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll"]
-git-tree-sha1 = "2839f1c1296940218e35df0bbb220f2a79686670"
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "66d72dc6fcc86352f01676e8f0f698562e60510f"
 uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91"
-version = "1.18.0+4"
+version = "1.23.0+0"
 
 [[XML2_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
@@ -1011,9 +1093,9 @@ version = "1.4.0+3"
 
 [[ZipFile]]
 deps = ["Libdl", "Printf", "Zlib_jll"]
-git-tree-sha1 = "c3a5637e27e914a7a445b8d0ad063d701931e9f7"
+git-tree-sha1 = "3593e69e469d2111389a9bd06bac1f3d730ac6de"
 uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
-version = "0.9.3"
+version = "0.9.4"
 
 [[Zlib_jll]]
 deps = ["Libdl"]
@@ -1027,27 +1109,31 @@ version = "1.5.0+0"
 
 [[Zygote]]
 deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"]
-git-tree-sha1 = "531474afbc343c3c7cb9b71c2771813c6defd550"
+git-tree-sha1 = "88a4d79f4e389456d5a90d79d53d1738860ef0a5"
 uuid = "e88e6eb3-aa80-5325-afca-941959d7151f"
-version = "0.6.14"
+version = "0.6.34"
 
 [[ZygoteRules]]
 deps = ["MacroTools"]
-git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7"
+git-tree-sha1 = "8c1a8e4dfacb1fd631745552c8db35d0deb09ea0"
 uuid = "700de1a5-db45-46bc-99cf-38207098b444"
-version = "0.2.1"
+version = "0.2.2"
 
 [[libass_jll]]
-deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "acc685bcf777b2202a904cdcb49ad34c2fa1880c"
+deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47"
 uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
-version = "0.14.0+4"
+version = "0.15.1+0"
+
+[[libblastrampoline_jll]]
+deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
+uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
 
 [[libfdk_aac_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "7a5780a0d9c6864184b3a2eeeb833a0c871f00ab"
+git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55"
 uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280"
-version = "0.1.6+4"
+version = "2.0.2+0"
 
 [[libpng_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
@@ -1057,9 +1143,9 @@ version = "1.6.38+0"
 
 [[libvorbis_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"]
-git-tree-sha1 = "c45f4e40e7aafe9d086379e5578947ec8b95a8fb"
+git-tree-sha1 = "b910cb81ef3fe6e78bf6acee440bda86fd6ae00c"
 uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a"
-version = "1.3.7+0"
+version = "1.3.7+1"
 
 [[nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]
@@ -1071,15 +1157,15 @@ uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 
 [[x264_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "d713c1ce4deac133e3334ee12f4adff07f81778f"
+git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2"
 uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a"
-version = "2020.7.14+2"
+version = "2021.5.5+0"
 
 [[x265_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "487da2f8f2f0c8ee0e83f39d13037d6bbf0a45ab"
+git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9"
 uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76"
-version = "3.0.0+3"
+version = "3.5.0+0"
 
 [[xkbcommon_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"]

From 70e91563b54986716b491682755f6fec8b9592ee Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 16:01:38 +0000
Subject: [PATCH 17/41] Formatter

---
 examples/train-kernel-parameters/script.jl | 43 ++++++++--------------
 1 file changed, 15 insertions(+), 28 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 384e74171..b9b04a535 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -32,9 +32,6 @@ nothing #hide
 ## scatter(x_train, y_train; lab="data")
 ## plot!(x_test, sinc; lab="true function")
 
-
-
-
 # ## Base Approach
 # The first option is to rebuild the parametrized kernel from a vector of parameters 
 # in each evaluation of the cost fuction. This is similar to the approach taken in 
@@ -92,7 +89,7 @@ loss(θ)
 # Computational cost for one step
 
 @benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = only((Zygote.gradient(loss, θt))) 
+    grads = only((Zygote.gradient(loss, θt)))
     Optimise.update!(optt, θt, grads)
 end
 
@@ -100,22 +97,21 @@ end
 
 ## anim = Animation()
 for i in 1:25
-    grads = only((Zygote.gradient(loss, θ))) 
+    grads = only((Zygote.gradient(loss, θ)))
     Optimise.update!(opt, θ, grads)
 end;
-    ## scatter(
-    ##     x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
-    ## )
-    ## plot!(x_test, sinc; lab="true function")
-    ## plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
-    ## frame(anim)
+## scatter(
+##     x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
+## )
+## plot!(x_test, sinc; lab="true function")
+## plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
+## frame(anim)
 ## end
 ## gif(anim)
 
 # Final loss
 loss(θ)
 
-
 # ## Using ParameterHandling.jl
 # Alternatively, we can use the [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl) package 
 # to handle the requirement that all kernel parameters should be positive. 
@@ -123,18 +119,14 @@ loss(θ)
 using ParameterHandling
 
 raw_initial_θ = (
-    k1 = positive(1.1),
-    k2 = positive(0.1),
-    k3 = positive(0.01),
-    noise_var=positive(0.001),
+    k1=positive(1.1), k2=positive(0.1), k3=positive(0.01), noise_var=positive(0.001)
 )
 
 flat_θ, unflatten = ParameterHandling.value_flatten(raw_initial_θ)
 nothing #hide
 
 function kernelcall(θ)
-    return (θ.k1 * SqExponentialKernel() + θ.k2 * Matern32Kernel()) ∘
-           ScaleTransform(θ.k3)
+    return (θ.k1 * SqExponentialKernel() + θ.k2 * Matern32Kernel()) ∘ ScaleTransform(θ.k3)
 end
 nothing #hide
 
@@ -163,7 +155,7 @@ nothing #hide
 # ### Cost per step
 
 @benchmark let θt = flat_θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = (Zygote.gradient(loss ∘ unflatten, θt))[1] 
+    grads = (Zygote.gradient(loss ∘ unflatten, θt))[1]
     Optimise.update!(optt, θt, grads)
 end
 
@@ -171,7 +163,7 @@ end
 
 opt = Optimise.ADAGrad(0.5)
 for i in 1:25
-    grads = (Zygote.gradient(loss ∘ unflatten, flat_θ))[1] 
+    grads = (Zygote.gradient(loss ∘ unflatten, flat_θ))[1]
     Optimise.update!(opt, flat_θ, grads)
 end
 nothing #hide
@@ -180,16 +172,13 @@ nothing #hide
 
 (loss ∘ unflatten)(flat_θ)
 
-
 # ## Flux.destructure
 # If don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function. 
 # Again, we need to ensure that the parameters are positive. Note that the `exp` function now has to be in a different position. 
 
+θ = [1.1, 0.1, 0.01, 0.001]
 
-θ = [1.1, 0.1, 0.01, 0.001] 
-
-kernel = (θ[1] * SqExponentialKernel() + θ[2] * Matern32Kernel()) ∘
-ScaleTransform(θ[3])
+kernel = (θ[1] * SqExponentialKernel() + θ[2] * Matern32Kernel()) ∘ ScaleTransform(θ[3])
 
 p, kernelc = Flux.destructure(kernel);
 
@@ -198,12 +187,10 @@ p, kernelc = Flux.destructure(kernel);
 
 function f(x, x_train, y_train, θ)
     k = kernelc(θ[1:3])
-    return kernelmatrix(k, x, x_train) *
-           ((kernelmatrix(k, x_train) + (θ[4]) * I) \ y_train)
+    return kernelmatrix(k, x, x_train) * ((kernelmatrix(k, x_train) + (θ[4]) * I) \ y_train)
 end
 nothing #hide
 
-
 # We define the loss based on the L2 norm both
 # for the loss and the regularization
 

From 6a9e756e9793a866326bff0bad22516081428d24 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 16:24:57 +0000
Subject: [PATCH 18/41] Some small updates

---
 examples/train-kernel-parameters/script.jl | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index b9b04a535..9ae4319f5 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -1,6 +1,6 @@
 # # Train Kernel Parameters
 
-# In this example we show the two main methods to perform regression on a kernel from KernelFunctions.jl.
+# In this example we show a few ways to perform regression on a kernel from KernelFunctions.jl.
 
 # We load KernelFunctions and some other packages
 
@@ -209,18 +209,19 @@ loss(θ)
 # Initialize optimizer 
 
 opt = Optimise.ADAGrad(0.5)
+nothing #hide
 
 # Cost for one step
 
 @benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = only((Zygote.gradient(loss, θt))) # We compute the gradients given the kernel parameters and regularization
+    grads = only((Zygote.gradient(loss, θt))) 
     Optimise.update!(optt, θt, grads)
 end
 
 # The optimization 
 
 for i in 1:25
-    grads = only((Zygote.gradient(loss, θ))) # We compute the gradients given the kernel parameters and regularization
+    grads = only((Zygote.gradient(loss, θ))) 
     Optimise.update!(opt, θ, grads)
 end
 nothing #hide

From 8ab495de0b323f5d00f1d22c69b04fcb5cbf2340 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 16:37:44 +0000
Subject: [PATCH 19/41] Add BenchmarkTools

---
 docs/Manifest.toml | 22 ++++++++++++++++++++--
 docs/Project.toml  |  1 +
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/docs/Manifest.toml b/docs/Manifest.toml
index 647ea24ad..2fa47e286 100644
--- a/docs/Manifest.toml
+++ b/docs/Manifest.toml
@@ -20,6 +20,12 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
+[[BenchmarkTools]]
+deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"]
+git-tree-sha1 = "940001114a0147b6e4d10624276d56d531dd9b49"
+uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+version = "1.2.2"
+
 [[ChainRulesCore]]
 deps = ["Compat", "LinearAlgebra", "SparseArrays"]
 git-tree-sha1 = "926870acb6cbcf029396f2f2de030282b6bc1941"
@@ -146,7 +152,7 @@ version = "0.21.2"
 deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Random", "Requires", "SpecialFunctions", "StatsBase", "TensorCore", "Test", "ZygoteRules"]
 path = ".."
 uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
-version = "0.10.27"
+version = "0.10.28"
 
 [[Kronecker]]
 deps = ["LinearAlgebra", "NamedDims", "SparseArrays", "StatsBase"]
@@ -174,7 +180,7 @@ uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [[LinearAlgebra]]
-deps = ["Libdl"]
+deps = ["Libdl", "libblastrampoline_jll"]
 uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [[LogExpFunctions]]
@@ -221,6 +227,10 @@ version = "0.2.43"
 [[NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 
+[[OpenBLAS_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
+uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
+
 [[OpenLibm_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
@@ -262,6 +272,10 @@ version = "1.2.3"
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
+[[Profile]]
+deps = ["Printf"]
+uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
+
 [[REPL]]
 deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
 uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
@@ -359,6 +373,10 @@ git-tree-sha1 = "8c1a8e4dfacb1fd631745552c8db35d0deb09ea0"
 uuid = "700de1a5-db45-46bc-99cf-38207098b444"
 version = "0.2.2"
 
+[[libblastrampoline_jll]]
+deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
+uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
+
 [[nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
diff --git a/docs/Project.toml b/docs/Project.toml
index c5d584e87..6929f1544 100644
--- a/docs/Project.toml
+++ b/docs/Project.toml
@@ -1,4 +1,5 @@
 [deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
 Kronecker = "2c470bb0-bcc8-11e8-3dad-c9649493f05e"

From 5f37f967d04683bb2ae53db59eefb39b90b66e9a Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 17:28:52 +0000
Subject: [PATCH 20/41] Move Benchmarks to the correct manifest

---
 docs/Manifest.toml                            | 10 -------
 docs/Project.toml                             |  1 -
 .../train-kernel-parameters/Manifest.toml     | 14 ++++++---
 examples/train-kernel-parameters/Project.toml |  2 ++
 examples/train-kernel-parameters/script.jl    | 30 +++++++++----------
 5 files changed, 27 insertions(+), 30 deletions(-)

diff --git a/docs/Manifest.toml b/docs/Manifest.toml
index 2fa47e286..9a5b760fe 100644
--- a/docs/Manifest.toml
+++ b/docs/Manifest.toml
@@ -20,12 +20,6 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
-[[BenchmarkTools]]
-deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"]
-git-tree-sha1 = "940001114a0147b6e4d10624276d56d531dd9b49"
-uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-version = "1.2.2"
-
 [[ChainRulesCore]]
 deps = ["Compat", "LinearAlgebra", "SparseArrays"]
 git-tree-sha1 = "926870acb6cbcf029396f2f2de030282b6bc1941"
@@ -272,10 +266,6 @@ version = "1.2.3"
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
-[[Profile]]
-deps = ["Printf"]
-uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
-
 [[REPL]]
 deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
 uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
diff --git a/docs/Project.toml b/docs/Project.toml
index 6929f1544..c5d584e87 100644
--- a/docs/Project.toml
+++ b/docs/Project.toml
@@ -1,5 +1,4 @@
 [deps]
-BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
 Kronecker = "2c470bb0-bcc8-11e8-3dad-c9649493f05e"
diff --git a/examples/train-kernel-parameters/Manifest.toml b/examples/train-kernel-parameters/Manifest.toml
index a8b46e154..a8aca047b 100644
--- a/examples/train-kernel-parameters/Manifest.toml
+++ b/examples/train-kernel-parameters/Manifest.toml
@@ -38,6 +38,12 @@ version = "0.2.0"
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
+[[BenchmarkTools]]
+deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"]
+git-tree-sha1 = "940001114a0147b6e4d10624276d56d531dd9b49"
+uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+version = "1.2.2"
+
 [[Bzip2_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
 git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2"
@@ -51,9 +57,9 @@ version = "0.4.1"
 
 [[CUDA]]
 deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"]
-git-tree-sha1 = "3eb49a30d6093dfb6cd60ea66ff681a7db87178a"
+git-tree-sha1 = "7405a853ebba81936827459bfe30608bbce7371e"
 uuid = "052768ef-5323-5732-b1bb-66c8b64840ba"
-version = "3.7.1"
+version = "3.8.0"
 
 [[Cairo_jll]]
 deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
@@ -696,9 +702,9 @@ version = "1.1.3"
 
 [[Plots]]
 deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun", "Unzip"]
-git-tree-sha1 = "7e4920a7d4323b8ffc3db184580598450bde8a8e"
+git-tree-sha1 = "db7393a80d0e5bef70f2b518990835541917a544"
 uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
-version = "1.25.7"
+version = "1.25.6"
 
 [[Preferences]]
 deps = ["TOML"]
diff --git a/examples/train-kernel-parameters/Project.toml b/examples/train-kernel-parameters/Project.toml
index 0645010d7..5d24101ce 100644
--- a/examples/train-kernel-parameters/Project.toml
+++ b/examples/train-kernel-parameters/Project.toml
@@ -1,4 +1,5 @@
 [deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
@@ -6,6 +7,7 @@ KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [compat]
 Distributions = "0.25"
diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 9ae4319f5..678c545e5 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -27,10 +27,11 @@ x_train = rand(Uniform(xmin, xmax), N) # We sample 100 random samples
 y_train = sinc.(x_train) + randn(N) * σ # We create a function and add some noise
 x_test = range(xmin - 0.1, xmax + 0.1; length=300)
 nothing #hide
+
 # Plot the data
 
-## scatter(x_train, y_train; lab="data")
-## plot!(x_test, sinc; lab="true function")
+scatter(x_train, y_train; lab="data")
+plot!(x_test, sinc; lab="true function")
 
 # ## Base Approach
 # The first option is to rebuild the parametrized kernel from a vector of parameters 
@@ -62,10 +63,10 @@ nothing #hide
 # with starting parameters [1.0, 1.0, 1.0, 1.0] we get :
 
 ŷ = f(x_test, x_train, y_train, log.(ones(4)))
-## scatter(x_train, y_train; lab="data")
-## plot!(x_test, sinc; lab="true function")
-## plot!(x_test, ŷ; lab="prediction")
-nothing #hide
+scatter(x_train, y_train; lab="data")
+plot!(x_test, sinc; lab="true function")
+plot!(x_test, ŷ; lab="prediction")
+
 
 # We define the loss based on the L2 norm both
 # for the loss and the regularization
@@ -99,15 +100,14 @@ end
 for i in 1:25
     grads = only((Zygote.gradient(loss, θ)))
     Optimise.update!(opt, θ, grads)
-end;
-## scatter(
-##     x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
-## )
-## plot!(x_test, sinc; lab="true function")
-## plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
-## frame(anim)
-## end
-## gif(anim)
+    scatter(
+        x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
+    )
+    plot!(x_test, sinc; lab="true function")
+    plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
+    frame(anim)
+end
+gif(anim)
 
 # Final loss
 loss(θ)

From 8fef513ca8ff366b7bedc3339d8078d31f1cb442 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 17:29:56 +0000
Subject: [PATCH 21/41] Formatter

---
 examples/train-kernel-parameters/script.jl | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 678c545e5..99761ce52 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -67,7 +67,6 @@ scatter(x_train, y_train; lab="data")
 plot!(x_test, sinc; lab="true function")
 plot!(x_test, ŷ; lab="prediction")
 
-
 # We define the loss based on the L2 norm both
 # for the loss and the regularization
 
@@ -214,14 +213,14 @@ nothing #hide
 # Cost for one step
 
 @benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = only((Zygote.gradient(loss, θt))) 
+    grads = only((Zygote.gradient(loss, θt)))
     Optimise.update!(optt, θt, grads)
 end
 
 # The optimization 
 
 for i in 1:25
-    grads = only((Zygote.gradient(loss, θ))) 
+    grads = only((Zygote.gradient(loss, θ)))
     Optimise.update!(opt, θ, grads)
 end
 nothing #hide

From 04d1ef6672e8e5113ce0059d6d2135f2c6d3a327 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 18:12:42 +0000
Subject: [PATCH 22/41] Fix missed comment

---
 examples/train-kernel-parameters/script.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 99761ce52..b4f4b8d2b 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -95,7 +95,7 @@ end
 
 # Optimizing
 
-## anim = Animation()
+anim = Animation()
 for i in 1:25
     grads = only((Zygote.gradient(loss, θ)))
     Optimise.update!(opt, θ, grads)

From 5a7cff4c5c687c7f224d8d11a35c169ea16eb4cc Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Fri, 28 Jan 2022 18:47:55 +0000
Subject: [PATCH 23/41] Add ParameterHandling too

---
 examples/train-kernel-parameters/Manifest.toml | 16 +++++++---------
 examples/train-kernel-parameters/Project.toml  |  1 +
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/examples/train-kernel-parameters/Manifest.toml b/examples/train-kernel-parameters/Manifest.toml
index a8aca047b..6efbdf10c 100644
--- a/examples/train-kernel-parameters/Manifest.toml
+++ b/examples/train-kernel-parameters/Manifest.toml
@@ -542,7 +542,7 @@ uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700"
 version = "2.36.0+0"
 
 [[LinearAlgebra]]
-deps = ["Libdl", "libblastrampoline_jll"]
+deps = ["Libdl"]
 uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [[Literate]]
@@ -629,10 +629,6 @@ git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f"
 uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051"
 version = "1.3.5+1"
 
-[[OpenBLAS_jll]]
-deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
-uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-
 [[OpenLibm_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
@@ -672,6 +668,12 @@ git-tree-sha1 = "ee26b350276c51697c9c2d88a072b339f9f03d73"
 uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
 version = "0.11.5"
 
+[[ParameterHandling]]
+deps = ["ChainRulesCore", "Compat", "InverseFunctions", "IterTools", "LinearAlgebra", "LogExpFunctions", "SparseArrays", "Test"]
+git-tree-sha1 = "1e38f118bf4c0428b64f39a0722446f17e95f82a"
+uuid = "2412ca09-6db7-441c-8e3a-88d5709968c5"
+version = "0.4.2"
+
 [[Parsers]]
 deps = ["Dates"]
 git-tree-sha1 = "0b5cfbb704034b5b4c1869e36634438a047df065"
@@ -1131,10 +1133,6 @@ git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47"
 uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
 version = "0.15.1+0"
 
-[[libblastrampoline_jll]]
-deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
-uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
-
 [[libfdk_aac_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
 git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55"
diff --git a/examples/train-kernel-parameters/Project.toml b/examples/train-kernel-parameters/Project.toml
index 5d24101ce..674c4c412 100644
--- a/examples/train-kernel-parameters/Project.toml
+++ b/examples/train-kernel-parameters/Project.toml
@@ -6,6 +6,7 @@ ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+ParameterHandling = "2412ca09-6db7-441c-8e3a-88d5709968c5"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 

From 862ffad7155f042092ce41b41d2665c610497e14 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sat, 29 Jan 2022 16:03:25 +0000
Subject: [PATCH 24/41] Change gif embed

---
 examples/train-kernel-parameters/script.jl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index b4f4b8d2b..0e9ffb6a1 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -106,7 +106,9 @@ for i in 1:25
     plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
     frame(anim)
 end
-gif(anim)
+gif(anim, "train-kernel-param.gif", fps = 15); nothing #hide
+
+![](train-kernel-param.gif)
 
 # Final loss
 loss(θ)

From b45ae7537d40034da09fa3b13aa13154e4c4215e Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sat, 29 Jan 2022 16:06:33 +0000
Subject: [PATCH 25/41] Forgot #

---
 examples/train-kernel-parameters/script.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 0e9ffb6a1..ff1b9c574 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -108,7 +108,7 @@ for i in 1:25
 end
 gif(anim, "train-kernel-param.gif", fps = 15); nothing #hide
 
-![](train-kernel-param.gif)
+# ![](train-kernel-param.gif)
 
 # Final loss
 loss(θ)

From 4091a3d871fc084055409f3776ebf9b936a1e1b4 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sat, 29 Jan 2022 16:10:17 +0000
Subject: [PATCH 26/41] Formatter

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 examples/train-kernel-parameters/script.jl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index ff1b9c574..42cd15533 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -106,7 +106,8 @@ for i in 1:25
     plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
     frame(anim)
 end
-gif(anim, "train-kernel-param.gif", fps = 15); nothing #hide
+gif(anim, "train-kernel-param.gif"; fps=15);
+nothing; #hide
 
 # ![](train-kernel-param.gif)
 

From d9d21dc4aa6130d8fcbb61a831dd7ecd295f694f Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sat, 29 Jan 2022 17:19:53 +0000
Subject: [PATCH 27/41] Compat and gif kwargs

---
 examples/train-kernel-parameters/Project.toml | 3 +++
 examples/train-kernel-parameters/script.jl    | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/Project.toml b/examples/train-kernel-parameters/Project.toml
index 674c4c412..3e4a65de0 100644
--- a/examples/train-kernel-parameters/Project.toml
+++ b/examples/train-kernel-parameters/Project.toml
@@ -11,10 +11,13 @@ Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [compat]
+BenchmarkTools = "1.2"
 Distributions = "0.25"
 Flux = "0.12"
 ForwardDiff = "0.10"
 KernelFunctions = "0.10"
 Literate = "2"
+ParameterHandling = "0.4"
 Plots = "1"
+Zygote = "0.6"
 julia = "1.3"
diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 42cd15533..0fa406e31 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -106,7 +106,7 @@ for i in 1:25
     plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
     frame(anim)
 end
-gif(anim, "train-kernel-param.gif"; fps=15);
+gif(anim, "train-kernel-param.gif"; fps=15; show_msg=false);
 nothing; #hide
 
 # ![](train-kernel-param.gif)

From 889cf4beb3e214be4b6d823a4306dc389f7be13b Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sat, 29 Jan 2022 17:40:59 +0000
Subject: [PATCH 28/41] , for ;

---
 examples/train-kernel-parameters/script.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 0fa406e31..e7af85539 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -106,7 +106,7 @@ for i in 1:25
     plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
     frame(anim)
 end
-gif(anim, "train-kernel-param.gif"; fps=15; show_msg=false);
+gif(anim, "train-kernel-param.gif"; fps=15, show_msg=false);
 nothing; #hide
 
 # ![](train-kernel-param.gif)

From 07308bdc37619a535f5f321ed0da7f06d8b8b7be Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sat, 29 Jan 2022 17:46:38 +0000
Subject: [PATCH 29/41] different commit msg

---
 examples/train-kernel-parameters/script.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index e7af85539..8eda1ec03 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -106,7 +106,7 @@ for i in 1:25
     plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
     frame(anim)
 end
-gif(anim, "train-kernel-param.gif"; fps=15, show_msg=false);
+gif(anim, "train-kernel-param.gif"; show_msg=false, fps=15);
 nothing; #hide
 
 # ![](train-kernel-param.gif)

From bd70d6c5c132a3fc980c80f5e3c2415b6aadb63d Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sat, 29 Jan 2022 19:20:14 +0000
Subject: [PATCH 30/41] Some more clarification

---
 examples/train-kernel-parameters/script.jl | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 8eda1ec03..93e01ec95 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -8,7 +8,6 @@ using KernelFunctions
 using LinearAlgebra
 using Distributions
 using Plots;
-default(; lw=2.0, legendfontsize=15.0);
 using BenchmarkTools
 using Flux
 using Flux: Optimise
@@ -176,7 +175,7 @@ nothing #hide
 
 # ## Flux.destructure
 # If don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function. 
-# Again, we need to ensure that the parameters are positive. Note that the `exp` function now has to be in a different position. 
+# Again, we need to ensure that the parameters are positive. Note that the `exp` function is now part of the loss function, instead of part of the kernel construction. 
 
 θ = [1.1, 0.1, 0.01, 0.001]
 
@@ -184,6 +183,9 @@ kernel = (θ[1] * SqExponentialKernel() + θ[2] * Matern32Kernel()) ∘ ScaleTra
 
 p, kernelc = Flux.destructure(kernel);
 
+# This returns the `trainable` parameters of the kernel and a function to reconstruct the kernel.
+kernelc(p) == kernel
+
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant
 

From a0f7bdc299f9fc3e5711dbb13dbad024cbd61ed6 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sat, 29 Jan 2022 21:20:19 +0000
Subject: [PATCH 31/41] Change display

---
 examples/train-kernel-parameters/script.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 93e01ec95..7b713c6e8 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -184,7 +184,7 @@ kernel = (θ[1] * SqExponentialKernel() + θ[2] * Matern32Kernel()) ∘ ScaleTra
 p, kernelc = Flux.destructure(kernel);
 
 # This returns the `trainable` parameters of the kernel and a function to reconstruct the kernel.
-kernelc(p) == kernel
+kernelc(p)
 
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant

From a9d6fa9dc89f55918bbce8c3554fa13b6a288488 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Wed, 2 Feb 2022 17:14:26 +0000
Subject: [PATCH 32/41] Final changes

---
 examples/train-kernel-parameters/script.jl | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 7b713c6e8..5aeeef22a 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -151,16 +151,16 @@ nothing #hide
 
 (loss ∘ unflatten)(flat_θ)
 
-# ## Training the model
+# ### Training the model
 
-# ### Cost per step
+# Cost per step
 
 @benchmark let θt = flat_θ[:], optt = Optimise.ADAGrad(0.5)
     grads = (Zygote.gradient(loss ∘ unflatten, θt))[1]
     Optimise.update!(optt, θt, grads)
 end
 
-# ### Complete optimization
+# Complete optimization
 
 opt = Optimise.ADAGrad(0.5)
 for i in 1:25
@@ -176,6 +176,7 @@ nothing #hide
 # ## Flux.destructure
 # If don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function. 
 # Again, we need to ensure that the parameters are positive. Note that the `exp` function is now part of the loss function, instead of part of the kernel construction. 
+# We could also use ParameterHandling.jl here, similar to the example above. 
 
 θ = [1.1, 0.1, 0.01, 0.001]
 
@@ -204,7 +205,7 @@ function loss(θ)
 end
 nothing #hide
 
-# ## Training the model
+# ### Training the model
 
 # The loss with our starting point :
 θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector

From 3dc7466095a526642abd334dc3a889788d393844 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Thu, 17 Feb 2022 14:41:50 +0000
Subject: [PATCH 33/41] Apply suggestions from code review

Co-authored-by: st-- <st--@users.noreply.github.com>
---
 examples/train-kernel-parameters/script.jl | 23 +++++++++++-----------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 5aeeef22a..452c0232a 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -1,13 +1,13 @@
 # # Train Kernel Parameters
 
-# In this example we show a few ways to perform regression on a kernel from KernelFunctions.jl.
+# Here we show a few ways to train (optimize) the kernel (hyper)parameters at the example of kernel-based regression using KernelFunctions.jl.
 
 # We load KernelFunctions and some other packages
 
 using KernelFunctions
 using LinearAlgebra
 using Distributions
-using Plots;
+using Plots
 using BenchmarkTools
 using Flux
 using Flux: Optimise
@@ -16,28 +16,27 @@ using Random: seed!
 seed!(42);
 
 # ## Data Generation
-# We generated data in 1 dimension
+# We generate a toy dataset in 1 dimension:
 
-xmin = -3;
-xmax = 3; # Bounds of the data
+xmin, xmax = -3, 3  # Bounds of the data
 N = 50 # Number of samples
-x_train = rand(Uniform(xmin, xmax), N) # We sample 100 random samples
+x_train = rand(Uniform(xmin, xmax), N)  # sample the inputs
 σ = 0.1
-y_train = sinc.(x_train) + randn(N) * σ # We create a function and add some noise
+y_train = sinc.(x_train) + randn(N) * σ  # evaluate a function and add some noise
 x_test = range(xmin - 0.1, xmax + 0.1; length=300)
 nothing #hide
 
 # Plot the data
 
-scatter(x_train, y_train; lab="data")
-plot!(x_test, sinc; lab="true function")
+scatter(x_train, y_train; label="data")
+plot!(x_test, sinc; label="true function")
 
-# ## Base Approach
+# ## Manual Approach
 # The first option is to rebuild the parametrized kernel from a vector of parameters 
 # in each evaluation of the cost fuction. This is similar to the approach taken in 
 # [Stheno.jl](https://github.com/JuliaGaussianProcesses/Stheno.jl).
 
-# To train the kernel parameters via ForwardDiff.jl
+# To train the kernel parameters via ForwardDiff.jl,
 # we need to create a function creating a kernel from an array.
 # A simple way to ensure that the kernel parameters are positive
 # is to optimize over the logarithm of the parameters. 
@@ -49,7 +48,7 @@ end
 nothing #hide
 
 # From theory we know the prediction for a test set x given
-# the kernel parameters and normalization constant
+# the kernel parameters and normalization constant:
 
 function f(x, x_train, y_train, θ)
     k = kernelcall(θ[1:3])

From b1711bbb15e9ff82cfe010431e795a65958aeec5 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Thu, 17 Feb 2022 15:36:31 +0000
Subject: [PATCH 34/41] Apply suggestions from code review

Co-authored-by: st-- <st--@users.noreply.github.com>
---
 examples/train-kernel-parameters/script.jl | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 452c0232a..30c5ef8bc 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -57,13 +57,13 @@ function f(x, x_train, y_train, θ)
 end
 nothing #hide
 
-# We look how the prediction looks like
-# with starting parameters [1.0, 1.0, 1.0, 1.0] we get :
+# Let's look at our prediction.
+# With starting parameters [1.0, 1.0, 1.0, 1.0] we get:
 
 ŷ = f(x_test, x_train, y_train, log.(ones(4)))
-scatter(x_train, y_train; lab="data")
-plot!(x_test, sinc; lab="true function")
-plot!(x_test, ŷ; lab="prediction")
+scatter(x_train, y_train; label="data")
+plot!(x_test, sinc; label="true function")
+plot!(x_test, ŷ; label="prediction")
 
 # We define the loss based on the L2 norm both
 # for the loss and the regularization
@@ -146,7 +146,7 @@ nothing #hide
 initial_θ = ParameterHandling.value(raw_initial_θ)
 nothing #hide
 
-# The loss with our starting point :
+# The loss at the initial parameter values:
 
 (loss ∘ unflatten)(flat_θ)
 
@@ -173,7 +173,7 @@ nothing #hide
 (loss ∘ unflatten)(flat_θ)
 
 # ## Flux.destructure
-# If don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function. 
+# If we don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function. 
 # Again, we need to ensure that the parameters are positive. Note that the `exp` function is now part of the loss function, instead of part of the kernel construction. 
 # We could also use ParameterHandling.jl here, similar to the example above. 
 
@@ -206,7 +206,7 @@ nothing #hide
 
 # ### Training the model
 
-# The loss with our starting point :
+# The loss at our initial parameter values:
 θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
 loss(θ)
 

From eaf6b8061199cccfac534b25be3b500dd9eb044d Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sun, 20 Mar 2022 00:15:57 +0000
Subject: [PATCH 35/41] Address comments

---
 examples/train-kernel-parameters/script.jl | 99 +++++++++++++---------
 1 file changed, 57 insertions(+), 42 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 30c5ef8bc..49ff221c6 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -1,8 +1,9 @@
 # # Train Kernel Parameters
 
-# Here we show a few ways to train (optimize) the kernel (hyper)parameters at the example of kernel-based regression using KernelFunctions.jl.
+# Here we show a few ways to train (optimize) the kernel (hyper)parameters at the example of kernel-based regression using KernelFunctions.jl. 
+# All options are functionally identical, but differ a little in readability, dependencies, and computational cost. 
 
-# We load KernelFunctions and some other packages
+# We load KernelFunctions and some other packages. Note that while we use `Zygote` for automatic differentiation and `Flux.optimise` for optimization, you should be able to replace them with your favourite autodiff framework or optimizer. 
 
 using KernelFunctions
 using LinearAlgebra
@@ -33,10 +34,10 @@ plot!(x_test, sinc; label="true function")
 
 # ## Manual Approach
 # The first option is to rebuild the parametrized kernel from a vector of parameters 
-# in each evaluation of the cost fuction. This is similar to the approach taken in 
+# in each evaluation of the cost function. This is similar to the approach taken in 
 # [Stheno.jl](https://github.com/JuliaGaussianProcesses/Stheno.jl).
 
-# To train the kernel parameters via ForwardDiff.jl,
+# To train the kernel parameters via [Zygote.jl](https://github.com/FluxML/Zygote.jl),
 # we need to create a function creating a kernel from an array.
 # A simple way to ensure that the kernel parameters are positive
 # is to optimize over the logarithm of the parameters. 
@@ -58,43 +59,48 @@ end
 nothing #hide
 
 # Let's look at our prediction.
-# With starting parameters [1.0, 1.0, 1.0, 1.0] we get:
+# With starting parameters `p0` (picked so we get the right local 
+# minimum for demonstration) we get:
 
-ŷ = f(x_test, x_train, y_train, log.(ones(4)))
+p0 = [1.1, 0.1, 0.01, 0.001]
+θ = log.(p0)
+ŷ = f(x_test, x_train, y_train, θ)
 scatter(x_train, y_train; label="data")
 plot!(x_test, sinc; label="true function")
 plot!(x_test, ŷ; label="prediction")
 
-# We define the loss based on the L2 norm both
-# for the loss and the regularization
+# We define the following loss:
 
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, θ)
-    return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
+    return norm(y_train - ŷ) + exp(θ[4]) * norm(ŷ)
 end
 nothing #hide
 
-# ### Training
-# Setting an initial value and initializing the optimizer: 
-θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
-opt = Optimise.ADAGrad(0.5)
-nothing #hide
-
 # The loss with our starting point:
 
 loss(θ)
 
-# Computational cost for one step
+# Computational cost for one step:
 
-@benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = only((Zygote.gradient(loss, θt)))
-    Optimise.update!(optt, θt, grads)
+@benchmark let 
+    θ = log.(p0)
+    opt = Optimise.ADAGrad(0.5)
+    grads = only((Zygote.gradient(loss, θ)))
+    Optimise.update!(opt, θ, grads)
 end
 
-# Optimizing
+# ### Training the model
+
+# Setting an initial value and initializing the optimizer: 
+θ = log.(p0) # Initial vector
+opt = Optimise.ADAGrad(0.5)
+nothing #hide
+
+# Optimize
 
 anim = Animation()
-for i in 1:25
+for i in 1:15
     grads = only((Zygote.gradient(loss, θ)))
     Optimise.update!(opt, θ, grads)
     scatter(
@@ -115,6 +121,8 @@ loss(θ)
 # ## Using ParameterHandling.jl
 # Alternatively, we can use the [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl) package 
 # to handle the requirement that all kernel parameters should be positive. 
+# The package also allows arbitrarily nesting named tuples that make the parameters 
+# more human readable, without having to remember their position in a flat vector. 
 
 using ParameterHandling
 
@@ -123,7 +131,9 @@ raw_initial_θ = (
 )
 
 flat_θ, unflatten = ParameterHandling.value_flatten(raw_initial_θ)
-nothing #hide
+flat_θ #hide
+
+# We define a few relevant functions and note that compared to the previous `kernelcall` function, we do not need explicit `exp`s. 
 
 function kernelcall(θ)
     return (θ.k1 * SqExponentialKernel() + θ.k2 * Matern32Kernel()) ∘ ScaleTransform(θ.k3)
@@ -139,7 +149,7 @@ nothing #hide
 
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, θ)
-    return sum(abs2, y_train - ŷ) + θ.noise_var * norm(ŷ)
+    return norm( y_train - ŷ) + θ.noise_var * norm(ŷ)
 end
 nothing #hide
 
@@ -150,19 +160,21 @@ nothing #hide
 
 (loss ∘ unflatten)(flat_θ)
 
-# ### Training the model
-
 # Cost per step
 
-@benchmark let θt = flat_θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = (Zygote.gradient(loss ∘ unflatten, θt))[1]
-    Optimise.update!(optt, θt, grads)
+@benchmark let 
+    θ = flat_θ[:]
+    opt = Optimise.ADAGrad(0.5)
+    grads = (Zygote.gradient(loss ∘ unflatten, θ))[1]
+    Optimise.update!(opt, θ, grads)
 end
 
-# Complete optimization
+# ### Training the model
+
+# Optimize
 
 opt = Optimise.ADAGrad(0.5)
-for i in 1:25
+for i in 1:15
     grads = (Zygote.gradient(loss ∘ unflatten, flat_θ))[1]
     Optimise.update!(opt, flat_θ, grads)
 end
@@ -172,10 +184,13 @@ nothing #hide
 
 (loss ∘ unflatten)(flat_θ)
 
+
 # ## Flux.destructure
 # If we don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function. 
 # Again, we need to ensure that the parameters are positive. Note that the `exp` function is now part of the loss function, instead of part of the kernel construction. 
-# We could also use ParameterHandling.jl here, similar to the example above. 
+
+# We could also use ParameterHandling.jl here. 
+# To do so, one would remove the `exp`s from the loss function below and call `loss ∘ unflatten` as above. 
 
 θ = [1.1, 0.1, 0.01, 0.001]
 
@@ -183,7 +198,7 @@ kernel = (θ[1] * SqExponentialKernel() + θ[2] * Matern32Kernel()) ∘ ScaleTra
 
 p, kernelc = Flux.destructure(kernel);
 
-# This returns the `trainable` parameters of the kernel and a function to reconstruct the kernel.
+# This returns the trainable `params` of the kernel and a function to reconstruct the kernel.
 kernelc(p)
 
 # From theory we know the prediction for a test set x given
@@ -200,10 +215,17 @@ nothing #hide
 
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, exp.(θ))
-    return sum(abs2, y_train - ŷ) + exp(θ[4]) * norm(ŷ)
+    return norm( y_train - ŷ) + exp(θ[4]) * norm(ŷ)
 end
 nothing #hide
 
+# Cost for one step
+
+@benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
+    grads = only((Zygote.gradient(loss, θt)))
+    Optimise.update!(optt, θt, grads)
+end
+
 # ### Training the model
 
 # The loss at our initial parameter values:
@@ -215,16 +237,9 @@ loss(θ)
 opt = Optimise.ADAGrad(0.5)
 nothing #hide
 
-# Cost for one step
-
-@benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
-    grads = only((Zygote.gradient(loss, θt)))
-    Optimise.update!(optt, θt, grads)
-end
-
-# The optimization 
+# Optimize
 
-for i in 1:25
+for i in 1:15
     grads = only((Zygote.gradient(loss, θ)))
     Optimise.update!(opt, θ, grads)
 end

From e6d5cc2dd0679e0565d1417cad46e0b9c9c4630e Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sun, 20 Mar 2022 00:19:27 +0000
Subject: [PATCH 36/41] Missed one

---
 examples/train-kernel-parameters/script.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 49ff221c6..c7dd4d989 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -196,10 +196,10 @@ nothing #hide
 
 kernel = (θ[1] * SqExponentialKernel() + θ[2] * Matern32Kernel()) ∘ ScaleTransform(θ[3])
 
-p, kernelc = Flux.destructure(kernel);
+params, kernelc = Flux.destructure(kernel);
 
 # This returns the trainable `params` of the kernel and a function to reconstruct the kernel.
-kernelc(p)
+kernelc(params)
 
 # From theory we know the prediction for a test set x given
 # the kernel parameters and normalization constant

From cc65ad52e9296b206d47d96633e0b2a6502098c8 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sun, 20 Mar 2022 00:22:56 +0000
Subject: [PATCH 37/41] Manifest issue

---
 docs/Manifest.toml | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/docs/Manifest.toml b/docs/Manifest.toml
index 7de7cf477..6fddd655e 100644
--- a/docs/Manifest.toml
+++ b/docs/Manifest.toml
@@ -149,11 +149,7 @@ version = "0.21.3"
 deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Random", "Requires", "SpecialFunctions", "StatsBase", "TensorCore", "Test", "ZygoteRules"]
 path = ".."
 uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
-<<<<<<< HEAD
-version = "0.10.28"
-=======
 version = "0.10.32"
->>>>>>> master
 
 [[deps.Kronecker]]
 deps = ["LinearAlgebra", "NamedDims", "SparseArrays", "StatsBase"]
@@ -180,11 +176,7 @@ uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
 [[deps.Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
-<<<<<<< HEAD
-[[LinearAlgebra]]
-=======
 [[deps.LinearAlgebra]]
->>>>>>> master
 deps = ["Libdl", "libblastrampoline_jll"]
 uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
@@ -232,19 +224,11 @@ version = "0.2.46"
 [[deps.NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 
-<<<<<<< HEAD
-[[OpenBLAS_jll]]
-deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
-uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-
-[[OpenLibm_jll]]
-=======
 [[deps.OpenBLAS_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
 uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
 
 [[deps.OpenLibm_jll]]
->>>>>>> master
 deps = ["Artifacts", "Libdl"]
 uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
 
@@ -383,19 +367,11 @@ git-tree-sha1 = "8c1a8e4dfacb1fd631745552c8db35d0deb09ea0"
 uuid = "700de1a5-db45-46bc-99cf-38207098b444"
 version = "0.2.2"
 
-<<<<<<< HEAD
-[[libblastrampoline_jll]]
-deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
-uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
-
-[[nghttp2_jll]]
-=======
 [[deps.libblastrampoline_jll]]
 deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
 uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
 
 [[deps.nghttp2_jll]]
->>>>>>> master
 deps = ["Artifacts", "Libdl"]
 uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
 

From 01bdb289aa8e653c765ca843b0d9dd143ee3fe56 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sun, 20 Mar 2022 00:25:56 +0000
Subject: [PATCH 38/41] Apply suggestions from formatter

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 examples/train-kernel-parameters/script.jl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index c7dd4d989..45815c8f5 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -83,7 +83,7 @@ loss(θ)
 
 # Computational cost for one step:
 
-@benchmark let 
+@benchmark let
     θ = log.(p0)
     opt = Optimise.ADAGrad(0.5)
     grads = only((Zygote.gradient(loss, θ)))
@@ -149,7 +149,7 @@ nothing #hide
 
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, θ)
-    return norm( y_train - ŷ) + θ.noise_var * norm(ŷ)
+    return norm(y_train - ŷ) + θ.noise_var * norm(ŷ)
 end
 nothing #hide
 
@@ -162,7 +162,7 @@ nothing #hide
 
 # Cost per step
 
-@benchmark let 
+@benchmark let
     θ = flat_θ[:]
     opt = Optimise.ADAGrad(0.5)
     grads = (Zygote.gradient(loss ∘ unflatten, θ))[1]
@@ -215,7 +215,7 @@ nothing #hide
 
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, exp.(θ))
-    return norm( y_train - ŷ) + exp(θ[4]) * norm(ŷ)
+    return norm(y_train - ŷ) + exp(θ[4]) * norm(ŷ)
 end
 nothing #hide
 

From b4f45321f00f985fffcdb21858f92bdc74f07a99 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Sun, 20 Mar 2022 00:30:21 +0000
Subject: [PATCH 39/41] Local formatter

---
 examples/train-kernel-parameters/script.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 45815c8f5..64567b319 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -184,7 +184,6 @@ nothing #hide
 
 (loss ∘ unflatten)(flat_θ)
 
-
 # ## Flux.destructure
 # If we don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function. 
 # Again, we need to ensure that the parameters are positive. Note that the `exp` function is now part of the loss function, instead of part of the kernel construction. 

From e0acaf4917e5ab06aaad5f542dae9f809e8d0387 Mon Sep 17 00:00:00 2001
From: Steffen Ridderbusch <steffen@robots.ox.ac.uk>
Date: Mon, 21 Mar 2022 12:00:43 +0000
Subject: [PATCH 40/41] Delete loss description.

Co-authored-by: st-- <st--@users.noreply.github.com>
---
 examples/train-kernel-parameters/script.jl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index 64567b319..d78f9a1f0 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -209,8 +209,6 @@ function f(x, x_train, y_train, θ)
 end
 nothing #hide
 
-# We define the loss based on the L2 norm both
-# for the loss and the regularization
 
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, exp.(θ))

From 04a5d988f457198fcbfea245ad353db41ffb80fd Mon Sep 17 00:00:00 2001
From: ST John <st--@users.noreply.github.com>
Date: Mon, 21 Mar 2022 15:21:22 +0200
Subject: [PATCH 41/41] format

---
 examples/train-kernel-parameters/script.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/train-kernel-parameters/script.jl b/examples/train-kernel-parameters/script.jl
index d78f9a1f0..85e35c5e2 100644
--- a/examples/train-kernel-parameters/script.jl
+++ b/examples/train-kernel-parameters/script.jl
@@ -209,7 +209,6 @@ function f(x, x_train, y_train, θ)
 end
 nothing #hide
 
-
 function loss(θ)
     ŷ = f(x_train, x_train, y_train, exp.(θ))
     return norm(y_train - ŷ) + exp(θ[4]) * norm(ŷ)