tests: group tests into several link-farms (10 tests per farm)

flake-modules/dev/devshell.nix

@@ -7,6 +7,7 @@
       lib,
       pkgs,
       config,
+      self',
       system,
       ...
     }:
@@ -44,9 +45,27 @@
 
                     text = builtins.readFile ./launch-test.sh;
                   };
+
+                  tests =
+                    let
+                      checks' = self'.checks;
+                      names = builtins.filter (n: builtins.match "test-.*" n != null) (builtins.attrNames checks');
+                    in
+                    builtins.listToAttrs (
+                      builtins.concatMap (
+                        checkName:
+                        map (testName: {
+                          name = testName;
+                          value = "${checkName}.passthru.entries.${testName}";
+                        }) (builtins.attrNames checks'.${checkName}.passthru.entries)
+                      ) names
+                    );
                 in
                 ''
-                  NIXVIM_SYSTEM=${system} NIXVIM_NIX_COMMAND=${nix} ${lib.getExe launchTest} "$@"
+                  export NIXVIM_SYSTEM=${system}
+                  export NIXVIM_NIX_COMMAND=${nix}
+                  export NIXVIM_TESTS=${pkgs.writers.writeJSON "tests.json" tests}
+                  ${lib.getExe launchTest} "$@"
                 '';
             }
             {

flake-modules/dev/launch-test.sh

@@ -36,8 +36,7 @@ nix_args=()
 interactive=false
 
 mk_test_list() {
-  nix eval ".#checks.${system}" --apply builtins.attrNames --json |
-    jq -r 'map(select(startswith("test-")))[]'
+  jq -r 'keys[]' "${NIXVIM_TESTS}"
 }
 
 while true; do
@@ -72,9 +71,26 @@ while true; do
   esac
 done
 
+get_tests() {
+  # Convert bash array to jq query
+  # e.g. (foo bar baz) => ."foo",."bar",."baz"
+  readarray -t queries < <(
+    for test in "$@"; do
+      echo '."'"$test"'"'
+    done
+  )
+  query=$(
+    IFS=,
+    echo "${queries[*]}"
+  )
+  for test in $(jq -r "${query}" "${NIXVIM_TESTS}"); do
+    echo "checks.${system}.${test}"
+  done
+}
+
 run_tests() {
-  # Add the prefix "checks.${system}." to each argument
-  if ! "${NIXVIM_NIX_COMMAND}" build "${nix_args[@]}" --no-link --file . "${@/#/checks.${system}.}"; then
+  readarray -t test_list < <(get_tests "$@")
+  if ! "${NIXVIM_NIX_COMMAND}" build "${nix_args[@]}" --no-link --file . "${test_list[@]}"; then
     echo "Test failure" >&2
     exit 1
   fi

lib/helpers.nix

@@ -71,6 +71,7 @@ let
       concatNonEmptyLines
       emptyTable
       enableExceptInTests
+      groupListBySize
       hasContent
       ifNonNull'
       listToUnkeyedAttrs

lib/utils.nix

@@ -147,4 +147,23 @@ rec {
       ${string}
       EOF
     '';
+
+  # Split a list into a several sub-list, each with a max-size of `size`
+  groupListBySize =
+    size: list:
+    reverseList (
+      foldl' (
+        lists: item:
+        let
+          first = head lists;
+          rest = drop 1 lists;
+        in
+        if lists == [ ] then
+          [ [ item ] ]
+        else if length first < size then
+          [ (first ++ [ item ]) ] ++ rest
+        else
+          [ [ item ] ] ++ lists
+      ) [ ] list
+    );
 }

tests/default.nix

@@ -37,10 +37,9 @@ let
   };
 in
 # We attempt to build & execute all configurations
-builtins.listToAttrs (
-  builtins.map (
-    { name, cases }:
-
+lib.pipe (testFiles ++ [ exampleFiles ]) [
+  (builtins.map (
+    file:
     let
       # The test case can either be the actual definition,
       # or a child attr named `module`.
@@ -59,8 +58,16 @@ builtins.listToAttrs (
         };
     in
     {
-      name = "test-${name}";
-      value = pkgs.linkFarm "test-${name}" (lib.map mkTest cases);
+      inherit (file) name;
+      path = pkgs.linkFarm file.name (builtins.map mkTest file.cases);
+    }
+  ))
+  (helpers.groupListBySize 10)
+  (lib.imap1 (
+    i: group: rec {
+      name = "test-${toString i}";
+      value = pkgs.linkFarm name group;
     }
-  ) (testFiles ++ [ exampleFiles ])
-)
+  ))
+  builtins.listToAttrs
+]

tests/lib-tests.nix

@@ -333,6 +333,47 @@ let
         "MIxEd"
       ];
     };
+
+    testGroupListBySize = {
+      expr = {
+        empty = helpers.groupListBySize 5 [ ];
+        "5/5" = helpers.groupListBySize 5 (lib.genList lib.id 5);
+        "13/5" = helpers.groupListBySize 5 (lib.genList lib.id 13);
+      };
+      expected = {
+        empty = [ ];
+        "5/5" = [
+          [
+            0
+            1
+            2
+            3
+            4
+          ]
+        ];
+        "13/5" = [
+          [
+            0
+            1
+            2
+            3
+            4
+          ]
+          [
+            5
+            6
+            7
+            8
+            9
+          ]
+          [
+            10
+            11
+            12
+          ]
+        ];
+      };
+    };
   };
 in
 if results == [ ] then
@@ -343,8 +384,8 @@ else
       results = pkgs.lib.concatStringsSep "\n" (
         builtins.map (result: ''
           ${result.name}:
-            expected: ${result.expected}
-            result: ${result.result}
+            expected: ${lib.generators.toPretty { } result.expected}
+            result: ${lib.generators.toPretty { } result.result}
         '') results
       );
     }