test: separate app and sandbox

Signed-off-by: Ophestra <cat@gensokyo.uk>

.gitea/workflows/test.yml

@@ -22,6 +22,57 @@ jobs:
           path: result/*
           retention-days: 1
 
+  race:
+    name: Fortify (race detector)
+    runs-on: nix
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run NixOS test
+        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.race
+
+      - name: Upload test output
+        uses: actions/upload-artifact@v3
+        with:
+          name: "fortify-race-vm-output"
+          path: result/*
+          retention-days: 1
+
+  sandbox:
+    name: Sandbox
+    runs-on: nix
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run NixOS test
+        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.sandbox
+
+      - name: Upload test output
+        uses: actions/upload-artifact@v3
+        with:
+          name: "sandbox-vm-output"
+          path: result/*
+          retention-days: 1
+
+  sandbox-race:
+    name: Sandbox (race detector)
+    runs-on: nix
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run NixOS test
+        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.sandbox-race
+
+      - name: Upload test output
+        uses: actions/upload-artifact@v3
+        with:
+          name: "sandbox-race-vm-output"
+          path: result/*
+          retention-days: 1
+
   fpkg:
     name: Fpkg
     runs-on: nix
@@ -39,23 +90,6 @@ jobs:
           path: result/*
           retention-days: 1
 
-  race:
-    name: Data race detector
-    runs-on: nix
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Run NixOS test
-        run: nix build --out-link "result" --print-out-paths --print-build-logs .#checks.x86_64-linux.race
-
-      - name: Upload test output
-        uses: actions/upload-artifact@v3
-        with:
-          name: "fortify-race-vm-output"
-          path: result/*
-          retention-days: 1
-
   check:
     name: Flake checks
     needs:

flake.nix

@@ -58,12 +58,19 @@
         in
         {
           fortify = callPackage ./test { inherit system self; };
-          fpkg = callPackage ./cmd/fpkg/test { inherit system self; };
           race = callPackage ./test {
             inherit system self;
             withRace = true;
           };
 
+          sandbox = callPackage ./test/sandbox { inherit system self; };
+          sandbox-race = callPackage ./test/sandbox {
+            inherit system self;
+            withRace = true;
+          };
+
+          fpkg = callPackage ./cmd/fpkg/test { inherit system self; };
+
           formatting = runCommandLocal "check-formatting" { nativeBuildInputs = [ nixfmt-rfc-style ]; } ''
             cd ${./.}
 

test/configuration.nix

@@ -4,10 +4,6 @@
   config,
   ...
 }:
-let
-  testProgram = pkgs.callPackage ./sandbox/tool/package.nix { inherit (config.environment.fortify.package) version; };
-  testCases = import ./sandbox/case lib testProgram;
-in
 {
   users.users = {
     alice = {
@@ -41,9 +37,6 @@ in
       # For D-Bus tests:
       libnotify
       mako
-
-      # For checking seccomp outcome:
-      testProgram
     ];
 
     variables = {
@@ -109,10 +102,6 @@ in
     home-manager = _: _: { home.stateVersion = "23.05"; };
 
     apps = [
-      testCases.preset
-      testCases.tty
-      testCases.mapuid
-
       {
         name = "ne-foot";
         verbose = true;

test/sandbox/case/mapuid.nix

@@ -97,7 +97,6 @@
         "pki" = fs "80001ff" null null;
         "polkit-1" = fs "80001ff" null null;
         "profile" = fs "80001ff" null null;
-        "profiles" = fs "80001ff" null null;
         "protocols" = fs "80001ff" null null;
         "resolv.conf" = fs "80001ff" null null;
         "resolvconf.conf" = fs "80001ff" null null;

test/sandbox/case/preset.nix

@@ -97,7 +97,6 @@
         "pki" = fs "80001ff" null null;
         "polkit-1" = fs "80001ff" null null;
         "profile" = fs "80001ff" null null;
-        "profiles" = fs "80001ff" null null;
         "protocols" = fs "80001ff" null null;
         "resolv.conf" = fs "80001ff" null null;
         "resolvconf.conf" = fs "80001ff" null null;

test/sandbox/case/tty.nix

@@ -98,7 +98,6 @@
         "pki" = fs "80001ff" null null;
         "polkit-1" = fs "80001ff" null null;
         "profile" = fs "80001ff" null null;
-        "profiles" = fs "80001ff" null null;
         "protocols" = fs "80001ff" null null;
         "resolv.conf" = fs "80001ff" null null;
         "resolvconf.conf" = fs "80001ff" null null;

test/sandbox/configuration.nix

@@ -0,0 +1,76 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+let
+  testProgram = pkgs.callPackage ./tool/package.nix { inherit (config.environment.fortify.package) version; };
+  testCases = import ./case lib testProgram;
+in
+{
+  users.users = {
+    alice = {
+      isNormalUser = true;
+      description = "Alice Foobar";
+      password = "foobar";
+      uid = 1000;
+    };
+  };
+
+  home-manager.users.alice.home.stateVersion = "24.11";
+
+  # Automatically login on tty1 as a normal user:
+  services.getty.autologinUser = "alice";
+
+  environment = {
+    systemPackages = with pkgs; [
+      # For checking seccomp outcome:
+      testProgram
+    ];
+
+    variables = {
+      SWAYSOCK = "/tmp/sway-ipc.sock";
+      WLR_RENDERER = "pixman";
+    };
+  };
+
+  # Automatically configure and start Sway when logging in on tty1:
+  programs.bash.loginShellInit = ''
+    if [ "$(tty)" = "/dev/tty1" ]; then
+      set -e
+
+      mkdir -p ~/.config/sway
+      (sed s/Mod4/Mod1/ /etc/sway/config &&
+      echo 'output * bg ${pkgs.nixos-artwork.wallpapers.simple-light-gray.gnomeFilePath} fill' &&
+      echo 'output Virtual-1 res 1680x1050') > ~/.config/sway/config
+
+      sway --validate
+      systemd-cat --identifier=session sway && touch /tmp/sway-exit-ok
+    fi
+  '';
+
+  programs.sway.enable = true;
+
+  virtualisation.qemu.options = [
+    # Need to switch to a different GPU driver than the default one (-vga std) so that Sway can launch:
+    "-vga none -device virtio-gpu-pci"
+
+    # Increase performance:
+    "-smp 8"
+  ];
+
+  environment.fortify = {
+    enable = true;
+    stateDir = "/var/lib/fortify";
+    users.alice = 0;
+
+    home-manager = _: _: { home.stateVersion = "23.05"; };
+
+    apps = [
+      testCases.preset
+      testCases.tty
+      testCases.mapuid
+    ];
+  };
+}

test/sandbox/default.nix

@@ -0,0 +1,39 @@
+{
+  lib,
+  nixosTest,
+
+  self,
+  withRace ? false,
+}:
+
+nixosTest {
+  name = "fortify-sandbox" + (if withRace then "-race" else "");
+  nodes.machine =
+    { options, pkgs, ... }:
+    {
+      # Run with Go race detector:
+      environment.fortify = lib.mkIf withRace rec {
+        # race detector does not support static linking
+        package = (pkgs.callPackage ../../package.nix { }).overrideAttrs (previousAttrs: {
+          GOFLAGS = previousAttrs.GOFLAGS ++ [ "-race" ];
+        });
+        fsuPackage = options.environment.fortify.fsuPackage.default.override { fortify = package; };
+      };
+
+      imports = [
+        ./configuration.nix
+
+        self.nixosModules.fortify
+        self.inputs.home-manager.nixosModules.home-manager
+      ];
+    };
+
+  # adapted from nixos sway integration tests
+
+  # testScriptWithTypes:49: error: Cannot call function of unknown type
+  #           (machine.succeed if succeed else machine.execute)(
+  #           ^
+  # Found 1 error in 1 file (checked 1 source file)
+  skipTypeCheck = true;
+  testScript = builtins.readFile ./test.py;
+}

test/sandbox/test.py

@@ -0,0 +1,71 @@
+import json
+import shlex
+
+q = shlex.quote
+
+
+def swaymsg(command: str = "", succeed=True, type="command"):
+    assert command != "" or type != "command", "Must specify command or type"
+    shell = q(f"swaymsg -t {q(type)} -- {q(command)}")
+    with machine.nested(
+            f"sending swaymsg {shell!r}" + " (allowed to fail)" * (not succeed)
+    ):
+        ret = (machine.succeed if succeed else machine.execute)(
+            f"su - alice -c {shell}"
+        )
+
+    # execute also returns a status code, but disregard.
+    if not succeed:
+        _, ret = ret
+
+    if not succeed and not ret:
+        return None
+
+    parsed = json.loads(ret)
+    return parsed
+
+
+start_all()
+machine.wait_for_unit("multi-user.target")
+
+# To check fortify's version:
+print(machine.succeed("sudo -u alice -i fortify version"))
+
+# Wait for Sway to complete startup:
+machine.wait_for_file("/run/user/1000/wayland-1")
+machine.wait_for_file("/tmp/sway-ipc.sock")
+
+# Check seccomp outcome:
+swaymsg("exec fortify run cat")
+pid = int(machine.wait_until_succeeds("pgrep -U 1000000 -x cat", timeout=5))
+print(machine.succeed(f"fortify-test filter {pid} c698b081ff957afe17a6d94374537d37f2a63f6f9dd75da7546542407a9e32476ebda3312ba7785d7f618542bcfaf27ca27dcc2dddba852069d28bcfe8cad39a &>/dev/stdout", timeout=5))
+machine.succeed(f"kill -TERM {pid}")
+
+# Verify capabilities/securebits in user namespace:
+print(machine.succeed("sudo -u alice -i fortify run capsh --print"))
+print(machine.succeed("sudo -u alice -i fortify run capsh --has-no-new-privs"))
+print(machine.fail("sudo -u alice -i fortify run capsh --has-a=CAP_SYS_ADMIN"))
+print(machine.fail("sudo -u alice -i fortify run capsh --has-b=CAP_SYS_ADMIN"))
+print(machine.fail("sudo -u alice -i fortify run capsh --has-i=CAP_SYS_ADMIN"))
+print(machine.fail("sudo -u alice -i fortify run capsh --has-p=CAP_SYS_ADMIN"))
+print(machine.fail("sudo -u alice -i fortify run umount -R /dev"))
+
+# Check sandbox outcome:
+check_offset = 0
+def check_sandbox(name):
+    global check_offset
+    check_offset += 1
+    swaymsg(f"exec script /dev/null -E always -qec check-sandbox-{name}")
+    machine.wait_for_file(f"/tmp/fortify.1000/tmpdir/{check_offset}/sandbox-ok", timeout=15)
+
+
+check_sandbox("preset")
+check_sandbox("tty")
+check_sandbox("mapuid")
+
+# Exit Sway and verify process exit status 0:
+swaymsg("exit", succeed=False)
+machine.wait_for_file("/tmp/sway-exit-ok")
+
+# Print fortify runDir contents:
+print(machine.succeed("find /run/user/1000/fortify"))

test/test.py

@@ -99,40 +99,15 @@ print(denyOutputVerbose)
 # Fail direct fsu call:
 print(machine.fail("sudo -u alice -i fsu"))
 
-# Check seccomp outcome:
-swaymsg("exec fortify run cat")
-pid = int(machine.wait_until_succeeds("pgrep -U 1000000 -x cat", timeout=5))
-print(machine.succeed(f"fortify-test filter {pid} c698b081ff957afe17a6d94374537d37f2a63f6f9dd75da7546542407a9e32476ebda3312ba7785d7f618542bcfaf27ca27dcc2dddba852069d28bcfe8cad39a &>/dev/stdout", timeout=5))
-machine.succeed(f"kill -TERM {pid}")
-
-# Verify capabilities/securebits in user namespace:
-print(machine.succeed("sudo -u alice -i fortify run capsh --print"))
-print(machine.succeed("sudo -u alice -i fortify run capsh --has-no-new-privs"))
-print(machine.fail("sudo -u alice -i fortify run capsh --has-a=CAP_SYS_ADMIN"))
-print(machine.fail("sudo -u alice -i fortify run capsh --has-b=CAP_SYS_ADMIN"))
-print(machine.fail("sudo -u alice -i fortify run capsh --has-i=CAP_SYS_ADMIN"))
-print(machine.fail("sudo -u alice -i fortify run capsh --has-p=CAP_SYS_ADMIN"))
-print(machine.fail("sudo -u alice -i fortify run umount -R /dev"))
-
 # Verify PrintBaseError behaviour:
 if denyOutput != "fsu: uid 1001 is not in the fsurc file\n":
     raise Exception(f"unexpected deny output:\n{denyOutput}")
 if denyOutputVerbose != "fsu: uid 1001 is not in the fsurc file\nfortify: *cannot obtain uid from fsu: permission denied\n":
     raise Exception(f"unexpected deny verbose output:\n{denyOutputVerbose}")
 
-# Check sandbox outcome:
 check_offset = 0
-def check_sandbox(name):
-    global check_offset
-    check_offset += 1
-    swaymsg(f"exec script /dev/null -E always -qec check-sandbox-{name}")
-    machine.wait_for_file(f"/tmp/fortify.1000/tmpdir/{check_offset}/sandbox-ok", timeout=15)
 
 
-check_sandbox("preset")
-check_sandbox("tty")
-check_sandbox("mapuid")
-
 def aid(offset):
     return 1+check_offset+offset