yonah/monstersirenfetch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cmd/msrfetch: preserve metadata order

Browse Source 
The composite maps are no longer needed as the new Album variant makes
more sense than them. They are also unordered while the base variants of
both endpoints are ordered. Composite is therefore only used for
validation in the current implementation.

Signed-off-by: Yonah <contrib@gensokyo.uk>
This commit is contained in:
Yonah
2025-09-18 21:32:42 +09:00
parent 526a0371a4
commit 0999ccd211
9 changed files with 2690 additions and 2676 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
cmd/msrfetch/enrich.go
View File

@@ -20,8 +20,8 @@ func init() {
"Path to file containing the response body of /api/albums")
flag.StringVar(&flagSongsPath, "s", "songs.json",
"Path to file containing the response body of /api/songs")
flag.StringVar(&flagOutputPath, "o", "composite.json",
"Path to write composite data")
flag.StringVar(&flagOutputPath, "o", "metadata.json",
"Path to write enriched metadata")
}
func mustEnrich(ctx context.Context) {
@@ -32,48 +32,53 @@ func mustEnrich(ctx context.Context) {
mustReadJSON(flagAlbumsPath, &albumsResponse)
mustReadJSON(flagSongsPath, &songsResponse)
n := new(netDirect)
for i := range albumsResponse.Data {
a := &albumsResponse.Data[i]
if !a.IsFull() {
if err := a.Enrich(ctx, n); err != nil {
log.Fatal(err)
}
log.Printf("enriched album %s (%s) with %d songs", a.CID.String(), a.Name, len(a.Songs))
} else {
log.Printf("skipped album %s (%s)", a.CID.String(), a.Name)
}
}
for i := range songsResponse.Data.List {
s := &songsResponse.Data.List[i]
if !s.IsFull() {
if err := s.Enrich(ctx, n); err != nil {
log.Fatal(err)
}
log.Printf("enriched song %s: %s (%s)", s.CID.String(), s.SourceURL, s.Name)
} else {
log.Printf("skipped song %s: %s (%s)", s.CID.String(), s.SourceURL, s.Name)
}
}
// consistency check: enriched songs match flattened songs
if c, err := monstersirenfetch.Flatten(albumsResponse.Data, songsResponse.Data); err != nil {
log.Fatal(err)
} else {
n := new(netDirect)
for _, ca := range c {
if ca.Album == nil {
log.Fatal("albums contains nil")
log.Fatal("albums contain nil")
}
if !ca.Album.IsFull() {
if err = ca.Album.Enrich(ctx, n); err != nil {
log.Fatal(err)
}
log.Printf("enriched album %s (%s) with %d songs", ca.Album.CID.String(), ca.Album.Name, len(ca.Songs))
} else {
log.Printf("skipped album %s (%s)", ca.Album.CID.String(), ca.Album.Name)
}
// consistency check: for later validating enriched songs against flatten
flattenSongs := make([]monstersirenfetch.AlbumSong, 0, len(ca.Songs))
for _, cs := range ca.Songs {
if cs == nil {
log.Fatal("songs contains nil")
log.Fatal("songs contain nil")
}
flattenSongs = append(flattenSongs, monstersirenfetch.AlbumSong{CID: cs.CID, Name: cs.Name, Artists: cs.Artists})
if !cs.IsFull() {
if err = cs.Enrich(ctx, n); err != nil {
log.Fatal(err)
}
log.Printf("enriched song %s: %s (%s)", cs.CID.String(), cs.SourceURL, cs.Name)
} else {
log.Printf("skipped song %s: %s (%s)", cs.CID.String(), cs.SourceURL, cs.Name)
}
}
// consistency check: enriched songs match flattened songs
slices.SortFunc(flattenSongs, func(a, b monstersirenfetch.AlbumSong) int { return int(a.CID - b.CID) })
enrichSongs := make([]monstersirenfetch.AlbumSong, len(ca.Album.Songs))
copy(enrichSongs, ca.Album.Songs)
slices.SortFunc(enrichSongs, func(a, b monstersirenfetch.AlbumSong) int { return int(a.CID - b.CID) })
if !slices.EqualFunc(flattenSongs, enrichSongs, func(a monstersirenfetch.AlbumSong, b monstersirenfetch.AlbumSong) bool {
return a.CID == b.CID && a.Name == b.Name && slices.Equal(a.Artists, b.Artists)
}) {
@@ -82,7 +87,11 @@ func mustEnrich(ctx context.Context) {
log.Printf("validated %d songs associated with album %s", len(enrichSongs), ca.Album.CID.String())
}
}
mustWriteJSON(flagOutputPath, c)
log.Println("composite data written to", flagOutputPath)
}
mustWriteJSON(flagOutputPath, &monstersirenfetch.Metadata{
Albums: albumsResponse.Data,
Songs: songsResponse.Data.List,
})
log.Println("metadata written to", flagOutputPath)
}

64
cmd/msrfetch/fetch.go
View File

@@ -27,48 +27,46 @@ func init() {
}
func mustFetch(ctx context.Context) {
var c monstersirenfetch.CompositeAlbumsMap
mustReadJSON(flagOutputPath, &c)
const (
invalidContainsNil = "invalid composite data"
invalidNotEnriched = "this composite is not enriched"
invalidContainsNil = "invalid metadata"
invalidNotEnriched = "this metadata is not enriched"
)
var metadata *monstersirenfetch.Metadata
mustReadJSON(flagOutputPath, &metadata)
if metadata == nil {
log.Fatal(invalidContainsNil)
}
var urls []string
for _, ca := range c {
if ca.Album == nil {
log.Fatal(invalidContainsNil)
}
if !ca.Album.IsFull() {
for i := range metadata.Albums {
a := &metadata.Albums[i]
if !a.IsFull() {
log.Fatal(invalidNotEnriched)
}
if ca.Album.CoverURL == "" {
log.Fatalf("album %s missing coverUrl", ca.Album.CID.String())
if a.CoverURL == "" {
log.Fatalf("album %s missing coverUrl", a.CID.String())
}
urls = append(urls, ca.Album.CoverURL)
if ca.Album.CoverDeURL != "" {
urls = append(urls, ca.Album.CoverDeURL)
urls = append(urls, a.CoverURL)
if a.CoverDeURL != "" {
urls = append(urls, a.CoverDeURL)
}
}
for i := range metadata.Songs {
s := &metadata.Songs[i]
if !s.IsFull() {
log.Fatal(invalidNotEnriched)
}
for _, cs := range ca.Songs {
if cs == nil {
log.Fatal(invalidContainsNil)
}
if !cs.IsFull() {
log.Fatal(invalidNotEnriched)
}
urls = append(urls, cs.SourceURL)
if cs.LyricURL != "" {
urls = append(urls, cs.LyricURL)
}
if cs.MvURL != "" {
urls = append(urls, cs.MvURL)
}
if cs.MvCoverURL != "" {
urls = append(urls, cs.MvCoverURL)
}
urls = append(urls, s.SourceURL)
if s.LyricURL != "" {
urls = append(urls, s.LyricURL)
}
if s.MvURL != "" {
urls = append(urls, s.MvURL)
}
if s.MvCoverURL != "" {
urls = append(urls, s.MvCoverURL)
}
}
slices.Sort(urls)