derivation: collect store paths

The implementation of the main hack this library does.

derivation.go

@@ -10,7 +10,7 @@ import (
 
 type (
 	// DerivationMap is the output of `nix derivation show`.
-	DerivationMap map[string]Derivation
+	DerivationMap map[string]*Derivation
 
 	// Derivation is a description of a derivation.
 	Derivation struct {

derivation_sample_getchoo_atlas_test.go

@@ -0,0 +1,11 @@
+package nixbuild_test
+
+import _ "embed"
+
+var (
+	//go:embed testdata/derivation/show_getchoo_atlas.json
+	getchooAtlasShow []byte
+
+	//go:embed testdata/derivation/collect_getchoo_atlas
+	getchooAtlasCollective string
+)

derivation_sample_getchoo_glados_test.go

@@ -0,0 +1,11 @@
+package nixbuild_test
+
+import _ "embed"
+
+var (
+	//go:embed testdata/derivation/show_getchoo_glados.json
+	getchooGladosShow []byte
+
+	//go:embed testdata/derivation/collect_getchoo_glados
+	getchooGladosCollective string
+)

derivation_sample_pluiedev_pappardelle_test.go

@@ -0,0 +1,11 @@
+package nixbuild_test
+
+import _ "embed"
+
+var (
+	//go:embed testdata/derivation/show_pluiedev_pappardelle.json
+	pluiedevPappardelleShow []byte
+
+	//go:embed testdata/derivation/collect_pluiedev_pappardelle
+	pluiedevPappardelleCollective string
+)

derivation_test.go

@@ -0,0 +1,36 @@
+package nixbuild_test
+
+import (
+	"encoding/json"
+	"slices"
+	"strings"
+	"testing"
+
+	"git.gensokyo.uk/yonah/nixbuild"
+)
+
+func TestCollectFromDerivations(t *testing.T) {
+	testCases := []struct {
+		name string
+		show []byte
+		want string
+	}{
+		{"getchoo atlas", getchooAtlasShow, getchooAtlasCollective},
+		{"getchoo glados", getchooGladosShow, getchooGladosCollective},
+		{"pluiedev pappardelle", pluiedevPappardelleShow, pluiedevPappardelleCollective},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var derivations nixbuild.DerivationMap
+			if err := json.Unmarshal(tc.show, &derivations); err != nil {
+				t.Fatalf("cannot unmarshal test data: %v", err)
+			}
+			got := nixbuild.CollectFromDerivations(derivations)
+			want := strings.Split(strings.TrimSpace(tc.want), "\n")
+			if !slices.Equal(got, want) {
+				t.Errorf("CollectFromDerivations:\n%s, want\n%s",
+					strings.Join(got, "\n"), strings.Join(want, "\n"))
+			}
+		})
+	}
+}

instantiated.go

@@ -116,7 +116,7 @@ func (d *InstantiatedDecoder) Decode() ([]string, error) {
 	for drv := range d.Instantiated() {
 		if len(instantiated) == cap(instantiated) {
 			// grow the buffer exponentially to minimise copies
-			instantiated = slices.Grow(instantiated, cap(instantiated)<<1)
+			instantiated = slices.Grow(instantiated, cap(instantiated))
 		}
 		instantiated = append(instantiated, drv)
 	}

instantiated_sample_getchoo_atlas_test.go

@@ -3,10 +3,10 @@ package nixbuild_test
 import _ "embed"
 
 var (
-	//go:embed testdata/getchoo_atlas
+	//go:embed testdata/instantiated/getchoo_atlas
 	getchooAtlasOut string
 
-	getchooAtlas = []string{
+	getchooAtlasInstantiated = []string{
 		"/nix/store/003rskqvv65a6371jhvzlxw0s7k1ibrp-security-framework-sys-2.9.1.drv",
 		"/nix/store/005pb8jsa8m04d2jbyybwnrib8564k5q-winapi-i686-pc-windows-gnu-0.4.0.drv",
 		"/nix/store/007qwryp7ak63rn7m21kfbrhmdanb12y-hatch_fancy_pypi_readme-24.1.0.tar.gz.drv",

instantiated_sample_getchoo_glados_test.go

@@ -3,10 +3,10 @@ package nixbuild_test
 import _ "embed"
 
 var (
-	//go:embed testdata/getchoo_glados
+	//go:embed testdata/instantiated/getchoo_glados
 	getchooGladosOut string
 
-	getchooGlados = []string{
+	getchooGladosInstantiated = []string{
 		"/nix/store/001hj9sv6867zyfqbxqkw73jlgaxb70g-gnome-online-accounts-3.54.3_fish-completions.drv",
 		"/nix/store/00379qrgvkn6n137fimn9dsi9mpzd39c-remark-retext-5.0.1.tgz.drv",
 		"/nix/store/008w3qhsrafwdaz1n7dww65fhn3xw9lj-bash52-022.drv",

instantiated_sample_pluiedev_pappardelle_test.go

@@ -3,10 +3,10 @@ package nixbuild_test
 import _ "embed"
 
 var (
-	//go:embed testdata/pluiedev_pappardelle
+	//go:embed testdata/instantiated/pluiedev_pappardelle
 	pluiedevPappardelleOut string
 
-	pluiedevPappardelle = []string{
+	pluiedevPappardelleInstantiated = []string{
 		"/nix/store/008w3qhsrafwdaz1n7dww65fhn3xw9lj-bash52-022.drv",
 		"/nix/store/00aa261rv9fnvv4c1n80vygz6sy3dnpd-pyside6-6.9.1.drv",
 		"/nix/store/00biz15rp9gjfrz89g3b0sy8wxl5jsdr-WWW-RobotRules-6.02.tar.gz.drv",

instantiated_test.go

@@ -37,9 +37,9 @@ func TestDecodeInstantiated(t *testing.T) {
 			"/nix/store/ysp83x9nrks28zkblqmnc1s1kb68dr69-gnu-config-2024-01-01.drv",
 		}, nil},
 
-		{"getchoo atlas", getchooAtlasOut, getchooAtlas, nil},
-		{"getchoo glados", getchooGladosOut, getchooGlados, nil},
-		{"pluiedev pappardelle", pluiedevPappardelleOut, pluiedevPappardelle, nil},
+		{"getchoo atlas", getchooAtlasOut, getchooAtlasInstantiated, nil},
+		{"getchoo glados", getchooGladosOut, getchooGladosInstantiated, nil},
+		{"pluiedev pappardelle", pluiedevPappardelleOut, pluiedevPappardelleInstantiated, nil},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {

util.go

@@ -0,0 +1,56 @@
+package nixbuild
+
+import (
+	"path"
+	"slices"
+)
+
+// this is pretty close to zero copy for the test case
+const collectFromDerivationsInitialCap = 1 << 17
+
+// CollectFromDerivations returns a deduplicated slice of store paths collected from derivations.
+func CollectFromDerivations(derivations DerivationMap) []string {
+	var setpointSize, growToCap int
+	collective := make([]string, 0, collectFromDerivationsInitialCap)
+	for p, drv := range derivations {
+		if drv == nil {
+			// should be unreachable
+			continue
+		}
+
+		// another exponentially growing buffer here because range over map is expensive
+		setpointSize = len(collective) + len(drv.InputSources) +
+			// only map keys
+			len(drv.InputDerivations) +
+			// only Output.Path
+			len(drv.Outputs) +
+			// self, builder
+			2
+		if setpointSize > cap(collective) {
+			growToCap = cap(collective) << 1
+		tryGrow:
+			if setpointSize > growToCap {
+				// unlikely to be reached
+				growToCap <<= 1
+				goto tryGrow
+			}
+
+			collective = slices.Grow(collective, growToCap-cap(collective))
+		}
+
+		collective = append(collective, drv.InputSources...)
+		for s := range drv.InputDerivations {
+			collective = append(collective, s)
+		}
+		for _, out := range drv.Outputs {
+			collective = append(collective, out.Path)
+		}
+		collective = append(collective, p)
+		if path.IsAbs(drv.Builder) { // so builtins don't get collected
+			collective = append(collective, drv.Builder)
+		}
+	}
+
+	slices.Sort(collective)
+	return slices.Compact(collective)
+}