Regression testing

.github/workflows/benchmarking-files/benchmarks/classification.yaml

@@ -0,0 +1,13 @@
+---
+
+# binary, 1000 instances, 23 features, 2 labels
+- name: credit-g
+  openml_task_id: 31
+
+# multi-label, 1080 instances, 857 features, 9 labels
+- name: cnae-9
+  openml_task_id: 9981
+
+# binary, highly imbalanced, 2109 instances, 22 features, 2 labels
+- name: kc1
+  openml_task_id: 3917

.github/workflows/benchmarking-files/benchmarks/regression.yaml

@@ -0,0 +1,13 @@
+---
+
+# Regression, 303 instances, 14 features, 1 target
+- name: cholesterol
+  openml_task_id: 2295
+
+# Regression, 345 instances, 6 features, 1 target
+- name: liver-disorders
+  openml_task_id: 52948
+
+# Regression, 1460 instances, 80 features, 1 target
+- name: house-prices-nominal
+  openml_task_id: 359951

.github/workflows/benchmarking-files/config.yaml

@@ -0,0 +1,17 @@
+---
+frameworks:
+
+  definition_file:  # this allows to add custom framework definitions (in {user}/frameworks.yaml) on top of the default ones.
+    - '{root}/resources/frameworks.yaml'
+    - '{user}/frameworks.yaml'
+
+benchmarks:
+
+  definition_dir:  # this allows to add custom benchmark definitions (under {user}/benchmarks) to the default ones.
+    - '{root}/resources/benchmarks'
+    - '{user}/benchmarks'
+
+  constraints_file: # this allows to add custom constraint definitions (in {user}/constraints.yaml) on top of the default ones.
+    - '{root}/resources/constraints.yaml'
+    - '{user}/constraints.yaml'
+

.github/workflows/benchmarking-files/constraints.yaml

@@ -0,0 +1,4 @@
+---
+10fold10min:
+  folds: 10
+  max_runtime_seconds: 600

.github/workflows/benchmarking-files/frameworks.yaml

@@ -0,0 +1,5 @@
+---
+# This file is unused but just marks the structure of what is created
+# in github action runners.
+# For generating baselines we require no additional files
+# For creating regression tests we dynamically create this file