hst/fs: implement overlay fstype

This finally exposes overlay mounts in the high level hakurei API.

Signed-off-by: Ophestra <cat@gensokyo.uk>

cmd/hakurei/print_test.go

@@ -49,6 +49,7 @@ func Test_printShowInstance(t *testing.T) {
 
 Filesystem
  w+ephemeral(-rwxr-xr-x):/tmp/
+ w*/nix/store:/mnt-root/nix/.rw-store/upper:/mnt-root/nix/.rw-store/work:/mnt-root/nix/.ro-store
  */nix/store
  */run/current-system
  */run/opengl-driver
@@ -127,6 +128,7 @@ App
 
 Filesystem
  w+ephemeral(-rwxr-xr-x):/tmp/
+ w*/nix/store:/mnt-root/nix/.rw-store/upper:/mnt-root/nix/.rw-store/work:/mnt-root/nix/.ro-store
  */nix/store
  */run/current-system
  */run/opengl-driver
@@ -278,6 +280,15 @@ App
           "write": true,
           "perm": 493
         },
+        {
+          "type": "overlay",
+          "dst": "/nix/store",
+          "lower": [
+            "/mnt-root/nix/.ro-store"
+          ],
+          "upper": "/mnt-root/nix/.rw-store/upper",
+          "work": "/mnt-root/nix/.rw-store/work"
+        },
         {
           "type": "bind",
           "src": "/nix/store"
@@ -413,6 +424,15 @@ App
         "write": true,
         "perm": 493
       },
+      {
+        "type": "overlay",
+        "dst": "/nix/store",
+        "lower": [
+          "/mnt-root/nix/.ro-store"
+        ],
+        "upper": "/mnt-root/nix/.rw-store/upper",
+        "work": "/mnt-root/nix/.rw-store/work"
+      },
       {
         "type": "bind",
         "src": "/nix/store"
@@ -602,6 +622,15 @@ func Test_printPs(t *testing.T) {
             "write": true,
             "perm": 493
           },
+          {
+            "type": "overlay",
+            "dst": "/nix/store",
+            "lower": [
+              "/mnt-root/nix/.ro-store"
+            ],
+            "upper": "/mnt-root/nix/.rw-store/upper",
+            "work": "/mnt-root/nix/.rw-store/work"
+          },
           {
             "type": "bind",
             "src": "/nix/store"

hst/fs.go

@@ -81,6 +81,12 @@ func (f *FilesystemConfigJSON) MarshalJSON() ([]byte, error) {
 			*FSEphemeral
 		}{fsType{FilesystemEphemeral}, cv}
 
+	case *FSOverlay:
+		v = &struct {
+			fsType
+			*FSOverlay
+		}{fsType{FilesystemOverlay}, cv}
+
 	default:
 		return nil, FSImplError{f.FilesystemConfig}
 	}
@@ -103,6 +109,9 @@ func (f *FilesystemConfigJSON) UnmarshalJSON(data []byte) error {
 	case FilesystemEphemeral:
 		*f = FilesystemConfigJSON{new(FSEphemeral)}
 
+	case FilesystemOverlay:
+		*f = FilesystemConfigJSON{new(FSOverlay)}
+
 	default:
 		return FSTypeError(t.Type)
 	}

hst/fs_test.go

@@ -35,6 +35,10 @@ func TestFilesystemConfigJSON(t *testing.T) {
 			hst.FSImplError{Value: stubFS{"ephemeral"}},
 			"\x00", "\x00"},
 
+		{"bad impl overlay", hst.FilesystemConfigJSON{FilesystemConfig: stubFS{"overlay"}},
+			hst.FSImplError{Value: stubFS{"overlay"}},
+			"\x00", "\x00"},
+
 		{"bind", hst.FilesystemConfigJSON{
 			FilesystemConfig: &hst.FSBind{
 				Dst:      m("/etc"),
@@ -55,6 +59,17 @@ func TestFilesystemConfigJSON(t *testing.T) {
 		}, nil,
 			`{"type":"ephemeral","dst":"/run/user/65534","write":true,"size":1024,"perm":448}`,
 			`{"fs":{"type":"ephemeral","dst":"/run/user/65534","write":true,"size":1024,"perm":448},"magic":3236757504}`},
+
+		{"overlay", hst.FilesystemConfigJSON{
+			FilesystemConfig: &hst.FSOverlay{
+				Dst:   m("/nix/store"),
+				Lower: ms("/mnt-root/nix/.ro-store"),
+				Upper: m("/mnt-root/nix/.rw-store/upper"),
+				Work:  m("/mnt-root/nix/.rw-store/work"),
+			},
+		}, nil,
+			`{"type":"overlay","dst":"/nix/store","lower":["/mnt-root/nix/.ro-store"],"upper":"/mnt-root/nix/.rw-store/upper","work":"/mnt-root/nix/.rw-store/work"}`,
+			`{"fs":{"type":"overlay","dst":"/nix/store","lower":["/mnt-root/nix/.ro-store"],"upper":"/mnt-root/nix/.rw-store/upper","work":"/mnt-root/nix/.rw-store/work"},"magic":3236757504}`},
 	}
 
 	for _, tc := range testCases {

hst/fsoverlay.go

@@ -0,0 +1,98 @@
+package hst
+
+import (
+	"encoding/gob"
+	"strings"
+
+	"hakurei.app/container"
+)
+
+func init() { gob.Register(new(FSOverlay)) }
+
+// FilesystemOverlay is the [FilesystemConfig.Type] name of an overlay mount point.
+const FilesystemOverlay = "overlay"
+
+// FSOverlay represents an overlay mount point.
+type FSOverlay struct {
+	// mount point in container
+	Dst *container.Absolute `json:"dst"`
+
+	// any filesystem, does not need to be on a writable filesystem, must not be nil
+	Lower []*container.Absolute `json:"lower"`
+	// the upperdir is normally on a writable filesystem, leave as nil to mount Lower readonly
+	Upper *container.Absolute `json:"upper,omitempty"`
+	// the workdir needs to be an empty directory on the same filesystem as Upper, must not be nil if Upper is populated
+	Work *container.Absolute `json:"work,omitempty"`
+}
+
+func (o *FSOverlay) Valid() bool {
+	if o == nil || o.Dst == nil {
+		return false
+	}
+
+	for _, a := range o.Lower {
+		if a == nil {
+			return false
+		}
+	}
+
+	if o.Upper != nil { // rw
+		return o.Work != nil && len(o.Lower) > 0
+	} else { // ro
+		return len(o.Lower) >= 2
+	}
+}
+
+func (o *FSOverlay) Target() *container.Absolute {
+	if !o.Valid() {
+		return nil
+	}
+	return o.Dst
+}
+
+func (o *FSOverlay) Host() []*container.Absolute {
+	if !o.Valid() {
+		return nil
+	}
+	p := make([]*container.Absolute, 0, 2+len(o.Lower))
+	if o.Upper != nil && o.Work != nil {
+		p = append(p, o.Upper, o.Work)
+	}
+	p = append(p, o.Lower...)
+	return p
+}
+
+func (o *FSOverlay) Apply(op *container.Ops) {
+	if !o.Valid() {
+		return
+	}
+
+	if o.Upper != nil && o.Work != nil { // rw
+		op.Overlay(o.Dst, o.Upper, o.Work, o.Lower...)
+	} else { // ro
+		op.OverlayReadonly(o.Dst, o.Lower...)
+	}
+}
+
+func (o *FSOverlay) String() string {
+	if !o.Valid() {
+		return "<invalid>"
+	}
+
+	lower := make([]string, len(o.Lower))
+	for i, a := range o.Lower {
+		lower[i] = container.EscapeOverlayDataSegment(a.String())
+	}
+
+	if o.Upper != nil && o.Work != nil {
+		return "w*" + strings.Join(append([]string{
+			container.EscapeOverlayDataSegment(o.Dst.String()),
+			container.EscapeOverlayDataSegment(o.Upper.String()),
+			container.EscapeOverlayDataSegment(o.Work.String())},
+			lower...), container.SpecialOverlayPath)
+	} else {
+		return "*" + strings.Join(append([]string{
+			container.EscapeOverlayDataSegment(o.Dst.String())},
+			lower...), container.SpecialOverlayPath)
+	}
+}

hst/fsoverlay_test.go

@@ -0,0 +1,50 @@
+package hst_test
+
+import (
+	"testing"
+
+	"hakurei.app/container"
+	"hakurei.app/hst"
+)
+
+func TestFSOverlay(t *testing.T) {
+	checkFs(t, []fsTestCase{
+		{"nil", (*hst.FSOverlay)(nil), false, nil, nil, nil, "<invalid>"},
+		{"nil lower", &hst.FSOverlay{Dst: m("/etc"), Lower: []*container.Absolute{nil}}, false, nil, nil, nil, "<invalid>"},
+		{"zero lower", &hst.FSOverlay{Dst: m("/etc"), Upper: m("/"), Work: m("/")}, false, nil, nil, nil, "<invalid>"},
+		{"zero lower ro", &hst.FSOverlay{Dst: m("/etc")}, false, nil, nil, nil, "<invalid>"},
+		{"short lower", &hst.FSOverlay{Dst: m("/etc"), Lower: ms("/etc")}, false, nil, nil, nil, "<invalid>"},
+
+		{"full", &hst.FSOverlay{
+			Dst:   m("/nix/store"),
+			Lower: ms("/mnt-root/nix/.ro-store"),
+			Upper: m("/mnt-root/nix/.rw-store/upper"),
+			Work:  m("/mnt-root/nix/.rw-store/work"),
+		}, true, container.Ops{&container.MountOverlayOp{
+			Target: m("/nix/store"),
+			Lower:  ms("/mnt-root/nix/.ro-store"),
+			Upper:  m("/mnt-root/nix/.rw-store/upper"),
+			Work:   m("/mnt-root/nix/.rw-store/work"),
+		}}, m("/nix/store"), ms("/mnt-root/nix/.rw-store/upper", "/mnt-root/nix/.rw-store/work", "/mnt-root/nix/.ro-store"),
+			"w*/nix/store:/mnt-root/nix/.rw-store/upper:/mnt-root/nix/.rw-store/work:/mnt-root/nix/.ro-store"},
+
+		{"ro", &hst.FSOverlay{
+			Dst:   m("/mnt/src"),
+			Lower: ms("/tmp/.src0", "/tmp/.src1"),
+		}, true, container.Ops{&container.MountOverlayOp{
+			Target: m("/mnt/src"),
+			Lower:  ms("/tmp/.src0", "/tmp/.src1"),
+		}}, m("/mnt/src"), ms("/tmp/.src0", "/tmp/.src1"),
+			"*/mnt/src:/tmp/.src0:/tmp/.src1"},
+
+		{"ro work", &hst.FSOverlay{
+			Dst:   m("/mnt/src"),
+			Lower: ms("/tmp/.src0", "/tmp/.src1"),
+			Work:  m("/tmp"),
+		}, true, container.Ops{&container.MountOverlayOp{
+			Target: m("/mnt/src"),
+			Lower:  ms("/tmp/.src0", "/tmp/.src1"),
+		}}, m("/mnt/src"), ms("/tmp/.src0", "/tmp/.src1"),
+			"*/mnt/src:/tmp/.src0:/tmp/.src1"},
+	})
+}

hst/template.go

@@ -79,6 +79,12 @@ func Template() *Config {
 			},
 			Filesystem: []FilesystemConfigJSON{
 				{&FSEphemeral{Dst: container.AbsFHSTmp, Write: true, Perm: 0755}},
+				{&FSOverlay{
+					Dst:   container.MustAbs("/nix/store"),
+					Lower: []*container.Absolute{container.MustAbs("/mnt-root/nix/.ro-store")},
+					Upper: container.MustAbs("/mnt-root/nix/.rw-store/upper"),
+					Work:  container.MustAbs("/mnt-root/nix/.rw-store/work"),
+				}},
 				{&FSBind{Src: container.MustAbs("/nix/store")}},
 				{&FSBind{Src: container.AbsFHSRun.Append("current-system")}},
 				{&FSBind{Src: container.AbsFHSRun.Append("opengl-driver")}},

hst/template_test.go

@@ -103,6 +103,15 @@ func TestTemplate(t *testing.T) {
 				"write": true,
 				"perm": 493
 			},
+			{
+				"type": "overlay",
+				"dst": "/nix/store",
+				"lower": [
+					"/mnt-root/nix/.ro-store"
+				],
+				"upper": "/mnt-root/nix/.rw-store/upper",
+				"work": "/mnt-root/nix/.rw-store/work"
+			},
 			{
 				"type": "bind",
 				"src": "/nix/store"

test/sandbox/case/device.nix

@@ -47,7 +47,10 @@ in
     ];
 
     fs = fs "dead" {
-      ".hakurei" = fs "800001ed" { } null;
+      ".hakurei" = fs "800001ed" {
+        ".ro-store" = fs "801001fd" null null;
+        store = fs "800001ff" null null;
+      } null;
       bin = fs "800001ed" { sh = fs "80001ff" null null; } null;
       dev = fs "800001ed" null null;
       etc = fs "800001ed" {
@@ -218,6 +221,8 @@ in
       (ent "/devices" "/sys/devices" "ro,nosuid,nodev,noexec,relatime" "sysfs" "sysfs" "rw")
       (ent "/dri" "/dev/dri" "rw,nosuid" "devtmpfs" "devtmpfs" ignore)
       (ent "/var/cache" "/var/cache" "rw,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")
+      (ent "/" "/.hakurei/.ro-store" "rw,relatime" "overlay" "overlay" "ro,lowerdir=/host/nix/.ro-store:/host/nix/.rw-store/upper,redirect_dir=nofollow,userxattr")
+      (ent "/" "/.hakurei/store" "rw,relatime" "overlay" "overlay" "rw,lowerdir=/host/nix/.ro-store:/host/nix/.rw-store/upper,upperdir=/host/tmp/.hakurei-store-rw/upper,workdir=/host/tmp/.hakurei-store-rw/work,redirect_dir=nofollow,userxattr")
       (ent "/etc" ignore "ro,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")
       (ent "/" "/run/user" "rw,nosuid,nodev,relatime" "tmpfs" "ephemeral" "rw,size=4k,mode=755,uid=1000004,gid=1000004")
       (ent "/tmp/hakurei.1000/runtime/4" "/run/user/65534" "rw,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")

test/sandbox/case/mapuid.nix

@@ -56,7 +56,10 @@ in
     ];
 
     fs = fs "dead" {
-      ".hakurei" = fs "800001ed" { } null;
+      ".hakurei" = fs "800001ed" {
+        ".ro-store" = fs "801001fd" null null;
+        store = fs "800001ff" null null;
+      } null;
       bin = fs "800001ed" { sh = fs "80001ff" null null; } null;
       dev = fs "800001ed" {
         core = fs "80001ff" null null;
@@ -248,6 +251,8 @@ in
       (ent "/devices" "/sys/devices" "ro,nosuid,nodev,noexec,relatime" "sysfs" "sysfs" "rw")
       (ent "/dri" "/dev/dri" "rw,nosuid" "devtmpfs" "devtmpfs" ignore)
       (ent "/var/cache" "/var/cache" "rw,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")
+      (ent "/" "/.hakurei/.ro-store" "rw,relatime" "overlay" "overlay" "ro,lowerdir=/host/nix/.ro-store:/host/nix/.rw-store/upper,redirect_dir=nofollow,userxattr")
+      (ent "/" "/.hakurei/store" "rw,relatime" "overlay" "overlay" "rw,lowerdir=/host/nix/.ro-store:/host/nix/.rw-store/upper,upperdir=/host/tmp/.hakurei-store-rw/upper,workdir=/host/tmp/.hakurei-store-rw/work,redirect_dir=nofollow,userxattr")
       (ent "/etc" ignore "ro,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")
       (ent "/" "/run/user" "rw,nosuid,nodev,relatime" "tmpfs" "ephemeral" "rw,size=4k,mode=755,uid=1000003,gid=1000003")
       (ent "/tmp/hakurei.1000/runtime/3" "/run/user/1000" "rw,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")

test/sandbox/case/tty.nix

@@ -56,7 +56,10 @@ in
     ];
 
     fs = fs "dead" {
-      ".hakurei" = fs "800001ed" { } null;
+      ".hakurei" = fs "800001ed" {
+        ".ro-store" = fs "801001fd" null null;
+        store = fs "800001ff" null null;
+      } null;
       bin = fs "800001ed" { sh = fs "80001ff" null null; } null;
       dev = fs "800001ed" {
         console = fs "4200190" null null;
@@ -250,6 +253,8 @@ in
       (ent "/devices" "/sys/devices" "ro,nosuid,nodev,noexec,relatime" "sysfs" "sysfs" "rw")
       (ent "/dri" "/dev/dri" "rw,nosuid" "devtmpfs" "devtmpfs" ignore)
       (ent "/var/cache" "/var/cache" "rw,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")
+      (ent "/" "/.hakurei/.ro-store" "rw,relatime" "overlay" "overlay" "ro,lowerdir=/host/nix/.ro-store:/host/nix/.rw-store/upper,redirect_dir=nofollow,userxattr")
+      (ent "/" "/.hakurei/store" "rw,relatime" "overlay" "overlay" "rw,lowerdir=/host/nix/.ro-store:/host/nix/.rw-store/upper,upperdir=/host/tmp/.hakurei-store-rw/upper,workdir=/host/tmp/.hakurei-store-rw/work,redirect_dir=nofollow,uuid=on,userxattr")
       (ent "/etc" ignore "ro,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")
       (ent "/" "/run/user" "rw,nosuid,nodev,relatime" "tmpfs" "ephemeral" "rw,size=4k,mode=755,uid=1000002,gid=1000002")
       (ent "/tmp/hakurei.1000/runtime/2" "/run/user/65534" "rw,nosuid,nodev,relatime" "ext4" "/dev/disk/by-label/nixos" "rw")

test/sandbox/configuration.nix

@@ -82,6 +82,24 @@ in
         src = "/var/cache";
         write = true;
       }
+      {
+        type = "overlay";
+        dst = "/.hakurei/.ro-store";
+        lower = [
+          "/nix/.ro-store"
+          "/nix/.rw-store/upper"
+        ];
+      }
+      {
+        type = "overlay";
+        dst = "/.hakurei/store";
+        lower = [
+          "/nix/.ro-store"
+          "/nix/.rw-store/upper"
+        ];
+        upper = "/tmp/.hakurei-store-rw/upper";
+        work = "/tmp/.hakurei-store-rw/work";
+      }
     ];
 
     inherit (testCases) apps;

test/sandbox/test.py

@@ -55,6 +55,7 @@ print(machine.fail("sudo -u alice -i hakurei run capsh --has-p=CAP_SYS_ADMIN"))
 print(machine.fail("sudo -u alice -i hakurei run umount -R /dev"))
 
 # Check sandbox outcome:
+machine.succeed("install -dm0777 /tmp/.hakurei-store-rw/{upper,work}")
 check_offset = 0
 def check_sandbox(name):
     global check_offset