cmd/msrfetch: preserve metadata order

The composite maps are no longer needed as the new Album variant makes
more sense than them. They are also unordered while the base variants of
both endpoints are ordered. Composite is therefore only used for
validation in the current implementation.

Signed-off-by: Yonah <contrib@gensokyo.uk>

cmd/msrfetch/enrich.go

@@ -20,8 +20,8 @@ func init() {
 		"Path to file containing the response body of /api/albums")
 	flag.StringVar(&flagSongsPath, "s", "songs.json",
 		"Path to file containing the response body of /api/songs")
-	flag.StringVar(&flagOutputPath, "o", "composite.json",
-		"Path to write composite data")
+	flag.StringVar(&flagOutputPath, "o", "metadata.json",
+		"Path to write enriched metadata")
 }
 
 func mustEnrich(ctx context.Context) {
@@ -32,48 +32,53 @@ func mustEnrich(ctx context.Context) {
 	mustReadJSON(flagAlbumsPath, &albumsResponse)
 	mustReadJSON(flagSongsPath, &songsResponse)
 
+	n := new(netDirect)
+	for i := range albumsResponse.Data {
+		a := &albumsResponse.Data[i]
+		if !a.IsFull() {
+			if err := a.Enrich(ctx, n); err != nil {
+				log.Fatal(err)
+			}
+			log.Printf("enriched album %s (%s) with %d songs", a.CID.String(), a.Name, len(a.Songs))
+		} else {
+			log.Printf("skipped album %s (%s)", a.CID.String(), a.Name)
+		}
+	}
+	for i := range songsResponse.Data.List {
+		s := &songsResponse.Data.List[i]
+		if !s.IsFull() {
+			if err := s.Enrich(ctx, n); err != nil {
+				log.Fatal(err)
+			}
+			log.Printf("enriched song %s: %s (%s)", s.CID.String(), s.SourceURL, s.Name)
+		} else {
+			log.Printf("skipped song %s: %s (%s)", s.CID.String(), s.SourceURL, s.Name)
+		}
+	}
+
+	// consistency check: enriched songs match flattened songs
 	if c, err := monstersirenfetch.Flatten(albumsResponse.Data, songsResponse.Data); err != nil {
 		log.Fatal(err)
 	} else {
-		n := new(netDirect)
 		for _, ca := range c {
 			if ca.Album == nil {
-				log.Fatal("albums contains nil")
+				log.Fatal("albums contain nil")
 			}
 
-			if !ca.Album.IsFull() {
-				if err = ca.Album.Enrich(ctx, n); err != nil {
-					log.Fatal(err)
-				}
-				log.Printf("enriched album %s (%s) with %d songs", ca.Album.CID.String(), ca.Album.Name, len(ca.Songs))
-			} else {
-				log.Printf("skipped album %s (%s)", ca.Album.CID.String(), ca.Album.Name)
-			}
-
-			// consistency check: for later validating enriched songs against flatten
 			flattenSongs := make([]monstersirenfetch.AlbumSong, 0, len(ca.Songs))
-
 			for _, cs := range ca.Songs {
 				if cs == nil {
-					log.Fatal("songs contains nil")
+					log.Fatal("songs contain nil")
 				}
 				flattenSongs = append(flattenSongs, monstersirenfetch.AlbumSong{CID: cs.CID, Name: cs.Name, Artists: cs.Artists})
 
-				if !cs.IsFull() {
-					if err = cs.Enrich(ctx, n); err != nil {
-						log.Fatal(err)
-					}
-					log.Printf("enriched song %s: %s (%s)", cs.CID.String(), cs.SourceURL, cs.Name)
-				} else {
-					log.Printf("skipped song %s: %s (%s)", cs.CID.String(), cs.SourceURL, cs.Name)
-				}
 			}
-
-			// consistency check: enriched songs match flattened songs
 			slices.SortFunc(flattenSongs, func(a, b monstersirenfetch.AlbumSong) int { return int(a.CID - b.CID) })
+
 			enrichSongs := make([]monstersirenfetch.AlbumSong, len(ca.Album.Songs))
 			copy(enrichSongs, ca.Album.Songs)
 			slices.SortFunc(enrichSongs, func(a, b monstersirenfetch.AlbumSong) int { return int(a.CID - b.CID) })
+
 			if !slices.EqualFunc(flattenSongs, enrichSongs, func(a monstersirenfetch.AlbumSong, b monstersirenfetch.AlbumSong) bool {
 				return a.CID == b.CID && a.Name == b.Name && slices.Equal(a.Artists, b.Artists)
 			}) {
@@ -82,7 +87,11 @@ func mustEnrich(ctx context.Context) {
 				log.Printf("validated %d songs associated with album %s", len(enrichSongs), ca.Album.CID.String())
 			}
 		}
-		mustWriteJSON(flagOutputPath, c)
-		log.Println("composite data written to", flagOutputPath)
 	}
+
+	mustWriteJSON(flagOutputPath, &monstersirenfetch.Metadata{
+		Albums: albumsResponse.Data,
+		Songs:  songsResponse.Data.List,
+	})
+	log.Println("metadata written to", flagOutputPath)
 }

cmd/msrfetch/fetch.go

@@ -27,48 +27,46 @@ func init() {
 }
 
 func mustFetch(ctx context.Context) {
-	var c monstersirenfetch.CompositeAlbumsMap
-	mustReadJSON(flagOutputPath, &c)
-
 	const (
-		invalidContainsNil = "invalid composite data"
-		invalidNotEnriched = "this composite is not enriched"
+		invalidContainsNil = "invalid metadata"
+		invalidNotEnriched = "this metadata is not enriched"
 	)
 
+	var metadata *monstersirenfetch.Metadata
+	mustReadJSON(flagOutputPath, &metadata)
+	if metadata == nil {
+		log.Fatal(invalidContainsNil)
+	}
+
 	var urls []string
-	for _, ca := range c {
-		if ca.Album == nil {
-			log.Fatal(invalidContainsNil)
-		}
-		if !ca.Album.IsFull() {
+	for i := range metadata.Albums {
+		a := &metadata.Albums[i]
+		if !a.IsFull() {
 			log.Fatal(invalidNotEnriched)
 		}
-		if ca.Album.CoverURL == "" {
-			log.Fatalf("album %s missing coverUrl", ca.Album.CID.String())
+		if a.CoverURL == "" {
+			log.Fatalf("album %s missing coverUrl", a.CID.String())
 		}
-		urls = append(urls, ca.Album.CoverURL)
-		if ca.Album.CoverDeURL != "" {
-			urls = append(urls, ca.Album.CoverDeURL)
+		urls = append(urls, a.CoverURL)
+		if a.CoverDeURL != "" {
+			urls = append(urls, a.CoverDeURL)
+		}
+	}
+	for i := range metadata.Songs {
+		s := &metadata.Songs[i]
+		if !s.IsFull() {
+			log.Fatal(invalidNotEnriched)
 		}
 
-		for _, cs := range ca.Songs {
-			if cs == nil {
-				log.Fatal(invalidContainsNil)
-			}
-			if !cs.IsFull() {
-				log.Fatal(invalidNotEnriched)
-			}
-
-			urls = append(urls, cs.SourceURL)
-			if cs.LyricURL != "" {
-				urls = append(urls, cs.LyricURL)
-			}
-			if cs.MvURL != "" {
-				urls = append(urls, cs.MvURL)
-			}
-			if cs.MvCoverURL != "" {
-				urls = append(urls, cs.MvCoverURL)
-			}
+		urls = append(urls, s.SourceURL)
+		if s.LyricURL != "" {
+			urls = append(urls, s.LyricURL)
+		}
+		if s.MvURL != "" {
+			urls = append(urls, s.MvURL)
+		}
+		if s.MvCoverURL != "" {
+			urls = append(urls, s.MvCoverURL)
 		}
 	}
 	slices.Sort(urls)

generic.go

@@ -33,6 +33,13 @@ type Net interface {
 	Get(ctx context.Context, url string) (body io.ReadCloser, contentLength int64, err error)
 }
 
+// Metadata represents metadata, often enriched, of the entire website.
+// This is only used externally and is never returned by the API.
+type Metadata struct {
+	Albums []Album `json:"albums"`
+	Songs  []Song  `json:"songs"`
+}
+
 // Response is a generic API response.
 type Response[T any] struct {
 	Code    int    `json:"code"`