Add CI Tests (#161)

* use systemd-run instead of machinectl

* fix systemd user sessions by launching through runuser

* skip mounting binfmt_misc if the kernel lacks support for it

* link syschdemd/installer to static location

* make check happy

* test tests

* add a simple installer test

* add second test for exit codes

* make tests responsible for checking exit code

* fix: add which to path

* overhaul actions workflow

* move checks out of flake.nix

* use GUID for image name

* Split flake checks into matrix job

* Check for side-effects

* reformat powershell scripts

* extend basic test

* use pester for tests

* "fix" (purposefully) failing test

* rename lib -> init

* Add test for systemd --user

* Add test for docker-native

* move release to separate workflow

* change downstream workflow path

* switch to a class

* Test running with different user shells

* Add lib implementation for Windows

* Add documentation for the tests

* readme: docker permissions

* remove empty files

.editorconfig

@@ -0,0 +1,11 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = false

.github/workflows/main.yml

@@ -1,78 +1,103 @@
-name: 'Build NixOS WSL tarball'
+name: 'CI'
 
-on: [push, pull_request, release]
+on:
+  push: {}
+  pull_request: {}
+  workflow_call: {}
 
 jobs:
-  build:
+  find-tests:
+    name: Find Tests 🔍
     runs-on: ubuntu-latest
+    outputs:
+      tests: ${{ steps.tests.outputs.tests }}
+      checks: ${{ steps.checks.outputs.checks }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
-          # Nix Flakes doesn't work on shallow clones
           fetch-depth: 0
 
-      - name: Install nix
-        uses: cachix/install-nix-action@v12
-        with:
-          install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20201221_9fab14a/install
-          # Configure Nix to enable flakes
-          extra_nix_config: |
-            experimental-features = nix-command flakes
+      - name: Install nix ❄️
+        uses: cachix/install-nix-action@v18
 
-      - name: Run checks
+      - name: Find tests 🔍
+        id: tests
         run: |
-          nix flake check
+          find tests -name '*.Tests.ps1' -print0 | perl -pe 's|(.*?)\x0|"\1",|g;s|,$||;s|(.*)|tests=[\1]|' >> $GITHUB_OUTPUT
 
-      - name: Build tarball
+      - name: Find checks 🔍
+        id: checks
         run: |
-          nix build '.#nixosConfigurations.mysystem.config.system.build.tarball'
+          nix-instantiate --json --eval --strict -E 'with builtins; attrNames (getFlake (toString ./.)).checks.${currentSystem}' | perl -pe 's|(.*)|checks=\1|' >>$GITHUB_OUTPUT
 
-      - name: Upload tarball
-        uses: actions/upload-artifact@v2
+  build:
+    name: Build 🛠️
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
         with:
-          name: rootfs
-          path: result/tarball/nixos-wsl-x86_64-linux.tar.gz
+          fetch-depth: 0
 
-      - name: Build installer
+      - name: Install nix ❄️
+        uses: cachix/install-nix-action@v18
+
+      - name: Build installer 🛠️
         run: |
           nix build '.#nixosConfigurations.mysystem.config.system.build.installer'
 
-      - name: Upload installer
-        uses: actions/upload-artifact@v2
+      - name: Upload installer 📤
+        uses: actions/upload-artifact@v3
         with:
           name: installer
           path: result/tarball/nixos-wsl-installer.tar.gz
 
-  release:
-    if: startsWith(github.ref, 'refs/tags/')
-    needs: build
+  checks:
+    name: Flake Check 📋
+    needs:
+      - find-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        check: ${{ fromJSON(needs.find-tests.outputs.checks) }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-
-      - uses: actions/download-artifact@v2
+        uses: actions/checkout@v3
         with:
-          name: rootfs
+          fetch-depth: 0
 
-      - uses: actions/download-artifact@v2
-        with:
-          name: installer
+      - name: Install nix ❄️
+        uses: cachix/install-nix-action@v18
 
-      - name: Generate checksums
+      - name: Run check 📋
         run: |
-          for x in *.tar.gz; do
-            sha256sum $x > ${x}.sha256
-          done
+          nix build -L --impure --expr "with builtins; (getFlake (toString ./.)).checks.\${currentSystem}.${{ matrix.check }}"
 
-      - name: Attach to release
-        uses: softprops/action-gh-release@v1
+  tests:
+    name: Test 🧪
+    needs:
+      - find-tests
+      - build
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJSON(needs.find-tests.outputs.tests) }}
+        os:
+          - ubuntu-latest
+          # - windows-latest # doesn't work due to lack of nested virtualization on the runners, hopefully this will work one day
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Download installer 📥
+        uses: actions/download-artifact@v3
         with:
-          files: |
-            nixos-wsl-x86_64-linux.tar.gz
-            nixos-wsl-x86_64-linux.tar.gz.sha256
-            nixos-wsl-installer.tar.gz
-            nixos-wsl-installer.tar.gz.sha256
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          name: installer
+
+      - name: Execute test 🧪
+        shell: pwsh
+        run: |
+          Invoke-Pester -Output Detailed ${{ matrix.test }}

.github/workflows/release.yml

@@ -0,0 +1,35 @@
+name: 'Release'
+
+on:
+  push:
+    tags: []
+
+jobs:
+  build:
+    name: Build 🛠️
+    uses: nix-community/nixos-wsl/.github/workflows/main.yml@main
+  release:
+    needs:
+      - build
+    name: Create Release 📢
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download installer 📥
+        uses: actions/download-artifact@v3
+        with:
+          name: installer
+
+      - name: Generate checksums 🔑
+        run: |
+          for x in *.tar.gz; do
+            sha256sum $x > ${x}.sha256
+          done
+
+      - name: Attach to release 📦
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            nixos-wsl-installer.tar.gz
+            nixos-wsl-installer.tar.gz.sha256
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.vscode/launch.json

@@ -0,0 +1,11 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "PowerShell: Invoke Pester",
+      "type": "PowerShell",
+      "request": "launch",
+      "script": "Invoke-Pester -Output Detailed",
+    }
+  ]
+}

checks/nixpkgs-fmt.nix

@@ -0,0 +1,8 @@
+{ runCommand
+, nixpkgs-fmt
+, ...
+}:
+runCommand "check-nixpkgs-fmt" { nativeBuildInputs = [ nixpkgs-fmt ]; } ''
+  nixpkgs-fmt --check ${./..}
+  touch $out
+''

checks/shfmt.nix

@@ -0,0 +1,8 @@
+{ runCommand
+, shfmt
+, ...
+}:
+runCommand "check-shfmt" { nativeBuildInputs = [ shfmt ]; } ''
+  shfmt -i 2 -d ${./../scripts}/*.sh
+  touch $out
+''

checks/side-effects.nix

@@ -0,0 +1,35 @@
+# Test that including the WSL module in a config does not change anything without enabling it
+
+{ inputs
+, system
+, emptyFile
+, ...
+}:
+let
+  configModule = { config, options, ... }: {
+    fileSystems."/" = {
+      device = "/dev/sda1";
+      fsType = "ext4";
+    };
+    boot.loader.grub.device = "nodev";
+    system.stateVersion = options.system.stateVersion.default;
+  };
+
+  cleanConfig = inputs.nixpkgs.lib.nixosSystem {
+    inherit system;
+    modules = [
+      configModule
+    ];
+  };
+  wslModuleConfig = inputs.nixpkgs.lib.nixosSystem {
+    inherit system;
+    modules = [
+      configModule
+      inputs.self.nixosModules.wsl
+    ];
+  };
+in
+# Check that both configs evaluate to the same derivation
+if cleanConfig.config.system.build.toplevel.outPath == wslModuleConfig.config.system.build.toplevel.outPath
+then emptyFile
+else throw "The WSL module introduces a side-effect even when not enabled!"

flake.nix

@@ -11,7 +11,7 @@
     };
   };
 
-  outputs = { self, nixpkgs, flake-utils, ... }:
+  outputs = inputs@{ self, nixpkgs, flake-utils, ... }:
     {
 
       nixosModules.wsl = {
@@ -41,13 +41,15 @@
           pkgs = import nixpkgs { inherit system; };
         in
         {
-          checks = {
-            check-format = pkgs.runCommand "check-format" { nativeBuildInputs = with pkgs; [ nixpkgs-fmt shfmt ]; } ''
-              nixpkgs-fmt --check ${./.}
-              shfmt -i 2 -d ${./scripts}/*.sh
-              mkdir $out # success
-            '';
-          };
+          checks =
+            let
+              args = { inherit inputs; };
+            in
+            {
+              nixpkgs-fmt = pkgs.callPackage ./checks/nixpkgs-fmt.nix args;
+              shfmt = pkgs.callPackage ./checks/shfmt.nix args;
+              side-effects = pkgs.callPackage ./checks/side-effects.nix args;
+            };
 
           devShell = pkgs.mkShell {
             RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}";

modules/build-tarball.nix

@@ -75,8 +75,10 @@ in
     # These options make no sense without the wsl-distro module anyway
 
     system.build.tarball = pkgs.callPackage "${nixpkgs}/nixos/lib/make-system-tarball.nix" {
-      # No contents, structure will be added by prepare script
-      contents = [ ];
+
+      contents = [
+        { source = config.users.users.root.shell; target = "/nix/nixos-wsl/entrypoint"; }
+      ];
 
       fileName = "nixos-wsl-${pkgs.hostPlatform.system}";
 

modules/installer.nix

@@ -56,6 +56,7 @@ with builtins; with lib; {
           { source = passwd; target = "/etc/passwd"; }
           { source = "${pkgs.busybox}/bin/busybox"; target = "/bin/sh"; }
           { source = "${pkgs.busybox}/bin/busybox"; target = "/bin/mount"; }
+          { source = "${installer}"; target = "/nix/nixos-wsl/entrypoint"; }
         ];
 
         extraCommands = pkgs.writeShellScript "prepare" ''

scripts/syschdemd.nix

@@ -31,6 +31,7 @@ let
     src = ./wrapper.sh;
     path = lib.makeSearchPath "" [
       "/run/wrappers/bin" # mount
+      "${gnugrep}/bin" # grep
       "${systemd}/lib/systemd" # systemd
     ];
   };

scripts/syschdemd.sh

@@ -101,7 +101,7 @@ main() {
     start_systemd
   fi
 
-  if [ $# -gt 0 ]; then
+  if [ $# -gt 1 ]; then # Ignore just -c without a command
     # wsl seems to prefix with "-c"
     shift
     command="$*"

scripts/wrapper.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euxo pipefail
 
-mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
+if grep -q binfmt_misc /proc/filesystems; then
+  mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
+fi
 
 exec systemd

tests/README.md

@@ -0,0 +1,49 @@
+# Tests
+
+This directory contains tests that are executed against a built installer tarball.
+The test are written using the [Pester](https://pester.dev/) testing framework
+
+## Execute Tests
+
+The tests can be executed on both Windows or Linux.
+
+### Windows
+
+Make sure that you are able to run a distro in WSL2 before trying to run the tests.
+Please note that the tests are not compatible with Windows PowerShell, but require the new [PowerShell Core](https://apps.microsoft.com/store/detail/powershell/9MZ1SNWT0N5D?hl=en-us&gl=us).
+
+### Linux
+
+Running the tests requires Docker and PowerShell to be installed on your system. Make sure that the user you are running the tests as has permissions to run docker containers and that it is possible to access the internet from inside docker containers.
+
+### Running the Tests
+
+If you haven't already, [install Pester](https://pester.dev/docs/introduction/installation/).  
+The tests require a `nixos-wsl-installer.tar.gz` to be present in the current working directory or in `./result/tarball`. Refer to the top-level readme on how to build it.  
+Once everything is in place, run the test by running the following in PowerShell at the root of this repo:
+
+```powershell
+Invoke-Pester -Output Detailed ./tests
+```
+
+
+## Writing Test
+
+Please refer to [the Pester documentation](https://pester.dev/docs/quick-start) on how to write new tests.
+
+Put this snippet at the start of your test file to gain access to the following libray functions:  
+(This assumes that your test is at the root of the `tests` directory)
+
+```powershell
+BeforeAll {
+  . $PSScriptRoot/lib/lib.ps1
+}
+```
+
+- `Install-Distro`: Creates a new NixOS-WSL instance, automatically selecting the appropriate runtime (WSL or Docker) for the host OS. Returns a new `Distro` object
+- A Distro object has the following methods:
+  - `Launch($command)`: Runs the specified command inside the container. Returns the command output
+  - `GetPath($path)`: Returns the path inside the container, that points to the specified file on the host.
+  - `InstallConfig($path)`: Installs a nix-file as the systems `configuration.nix`. The default configuration is moved to `base.nix`, so that it can be imported by the new config
+  - `Shutdown()`: End all processes running in the container
+  - `Uninstall()`: Stop and then delete the container from the system. This should be called in an AfterEach or AfterAll block, so that the test does not leave it on the system.