rosa/hakurei
6
4
Fork 2
Code Issues 12 Pull Requests 1 Actions Packages Projects Releases 21 Wiki Activity

internal/pkg: decompress xz streams
Test / Create distribution (push) Successful in 59s
Details
Test / Sandbox (push) Successful in 2m54s
Details
Test / Hakurei (push) Successful in 4m48s
Details
Test / Sandbox (race detector) (push) Successful in 5m59s
Details
Test / Hakurei (race detector) (push) Successful in 7m35s
Details
Test / ShareFS (push) Successful in 7m42s
Details
Test / Flake checks (push) Successful in 1m18s
Details

Browse Source 
This uses a port of the public domain xz implementation.

Signed-off-by: Ophestra <cat@gensokyo.uk>
This commit is contained in:
cat
2026-07-25 14:03:34 +09:00
parent e26914cd6f
commit e5abf51296
80 changed files with 4203 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/xz/reader_test.go
+710
View File
@@ -0,0 +1,710 @@
/*
* Package xz tests
*
* Author: Michael Cross <https://github.com/xi2>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
package xz_test
import (
"bytes"
"crypto/md5"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"testing/iotest"
"hakurei.app/internal/xz"
)
type testFile struct {
file string
md5sum string
err error
}
// Note that the md5sums below were generated with XZ Utils and
// XZ Embedded, not with this package.
var badFiles = []testFile{
{
file: "bad-0-backward_size.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-0cat-alone.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrFormat,
},
{
file: "bad-0cat-header_magic.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrFormat,
},
{
file: "bad-0catpad-empty.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-0-empty-truncated.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrBuf,
},
{
file: "bad-0-footer_magic.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-0-header_magic.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrFormat,
},
{
file: "bad-0-nonempty_index.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-0pad-empty.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-block_header-1.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-block_header-2.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-block_header-3.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-block_header-4.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-block_header-5.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-block_header-6.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-check-crc32.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-1-check-crc64.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-1-check-sha256.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-1-lzma2-1.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-lzma2-2.xz",
md5sum: "211dbb3d39f3c244585397f6d3c09be3",
err: xz.ErrData,
},
{
file: "bad-1-lzma2-3.xz",
md5sum: "211dbb3d39f3c244585397f6d3c09be3",
err: xz.ErrData,
},
{
file: "bad-1-lzma2-4.xz",
md5sum: "6492b8d167aee3ca222d07a49d24015a",
err: xz.ErrData,
},
{
file: "bad-1-lzma2-5.xz",
md5sum: "6492b8d167aee3ca222d07a49d24015a",
err: xz.ErrData,
},
{
file: "bad-1-lzma2-6.xz",
md5sum: "09f7e02f1290be211da707a266f153b3",
err: xz.ErrData,
},
{
file: "bad-1-lzma2-7.xz",
md5sum: "c214a5e586cb3f0673cc6138f7de25ab",
err: xz.ErrData,
},
{
file: "bad-1-lzma2-8.xz",
md5sum: "211dbb3d39f3c244585397f6d3c09be3",
err: xz.ErrData,
},
{
file: "bad-1-stream_flags-1.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-1-stream_flags-2.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-stream_flags-3.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-1-vli-1.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-1-vli-2.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrData,
},
{
file: "bad-2-compressed_data_padding.xz",
md5sum: "09f7e02f1290be211da707a266f153b3",
err: xz.ErrData,
},
{
file: "bad-2-index-1.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-2-index-2.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-2-index-3.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-2-index-4.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
{
file: "bad-2-index-5.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: xz.ErrData,
},
}
var goodFiles = []testFile{
{
file: "good-0cat-empty.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: nil,
},
{
file: "good-0catpad-empty.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: nil,
},
{
file: "good-0-empty.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: nil,
},
{
file: "good-0pad-empty.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: nil,
},
{
file: "good-1-3delta-lzma2.xz",
md5sum: "c214a5e586cb3f0673cc6138f7de25ab",
err: nil,
},
{
file: "good-1-block_header-1.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: nil,
},
{
file: "good-1-block_header-2.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: nil,
},
{
file: "good-1-block_header-3.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: nil,
},
{
file: "good-1-check-crc32.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: nil,
},
{
file: "good-1-check-crc64.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: nil,
},
{
file: "good-1-check-none.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: nil,
},
{
file: "good-1-check-sha256.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: nil,
},
{
file: "good-1-delta-lzma2.tiff.xz",
md5sum: "c692be6d1987989af5eeafc329085ad2",
err: nil,
},
{
file: "good-1-lzma2-1.xz",
md5sum: "c214a5e586cb3f0673cc6138f7de25ab",
err: nil,
},
{
file: "good-1-lzma2-2.xz",
md5sum: "c214a5e586cb3f0673cc6138f7de25ab",
err: nil,
},
{
file: "good-1-lzma2-3.xz",
md5sum: "c214a5e586cb3f0673cc6138f7de25ab",
err: nil,
},
{
file: "good-1-lzma2-4.xz",
md5sum: "c214a5e586cb3f0673cc6138f7de25ab",
err: nil,
},
{
file: "good-1-lzma2-5.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: nil,
},
{
file: "good-1-sparc-lzma2.xz",
md5sum: "835f2865f1d7c7ad2c7de0d5fd07faef",
err: nil,
},
{
file: "good-1-x86-lzma2.xz",
md5sum: "ce212d6a1cfe73d8395a2b42f94c2419",
err: nil,
},
{
file: "good-2-lzma2.xz",
md5sum: "fbf68a8e34b2ded53bba54e68794b4fe",
err: nil,
},
}
var unsupportedFiles = []testFile{
{
file: "unsupported-block_header.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrOptions,
},
{
file: "unsupported-check.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrUnsupportedCheck,
},
{
file: "unsupported-filter_flags-1.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrOptions,
},
{
file: "unsupported-filter_flags-2.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrOptions,
},
{
file: "unsupported-filter_flags-3.xz",
md5sum: "d41d8cd98f00b204e9800998ecf8427e",
err: xz.ErrOptions,
},
}
var otherFiles = []testFile{
{
file: "good-1-x86-lzma2-offset-2048.xz",
md5sum: "ce212d6a1cfe73d8395a2b42f94c2419",
err: nil,
},
{
file: "good-2-lzma2-corrupt.xz",
md5sum: "d9c5223e7e6e305e6c1c6ed73789df88",
err: xz.ErrData,
},
{
file: "words.xz",
md5sum: "00e28a90cb4a975fdaa3b375d3124a66",
err: nil,
},
{
file: "random-1mb.xz",
md5sum: "3f04b090e5d26a1cbeea53c21ebcad03",
err: nil,
},
{
file: "zeros-100mb.xz",
md5sum: "0f86d7c5a6180cf9584c1d21144d85b0",
err: nil,
},
}
func openTestFile(file string) (*os.File, error) {
f, err := os.Open(filepath.Join("testdata", "xz-utils", file))
if err == nil {
return f, nil
}
if err != nil && !os.IsNotExist(err) {
return nil, err
}
f, err = os.Open(filepath.Join("testdata", "other", file))
if err == nil {
return f, nil
}
return nil, err
}
func readTestFile(file string) ([]byte, error) {
b, err := ioutil.ReadFile(filepath.Join("testdata", "xz-utils", file))
if err == nil {
return b, nil
}
if err != nil && !os.IsNotExist(err) {
return nil, err
}
b, err = ioutil.ReadFile(filepath.Join("testdata", "other", file))
if err == nil {
return b, nil
}
return nil, err
}
// testFileData returns the data (md5sum and err) associated with the
// named testFile
func testFileData(file string) (md5sum string, err error) {
fileList := badFiles
fileList = append(fileList, goodFiles...)
fileList = append(fileList, unsupportedFiles...)
fileList = append(fileList, otherFiles...)
for _, f := range fileList {
if f.file == file {
return f.md5sum, f.err
}
}
return "", nil
}
// testFileList tests the decoding of a list of files against their
// expected error and md5sum.
func testFileList(t *testing.T, files []testFile, reuseReader bool) {
var r *xz.Reader
var err error
if reuseReader {
r, err = xz.NewReader(nil, 0)
}
for _, f := range files {
func() {
var fr *os.File
fr, err = openTestFile(f.file)
if err != nil {
t.Fatal(err)
}
defer fr.Close()
hash := md5.New()
switch reuseReader {
case true:
err = r.Reset(fr)
case false:
r, err = xz.NewReader(fr, 0)
}
if err == nil {
_, err = io.Copy(hash, r)
}
if err != f.err {
t.Fatalf("%s: wanted error: %v, got: %v\n", f.file, f.err, err)
}
md5sum := fmt.Sprintf("%x", hash.Sum(nil))
if f.md5sum != md5sum {
t.Fatalf(
"%s: wanted md5: %v, got: %v\n", f.file, f.md5sum, md5sum)
}
}()
}
}
// testFileListByteReads tests the decoding of a list of files against
// their expected error and md5sum. It uses a one byte input buffer
// and one byte output buffer for each run of the decoder.
func testFileListByteReads(t *testing.T, files []testFile) {
for _, f := range files {
func() {
fr, err := openTestFile(f.file)
if err != nil {
t.Fatal(err)
}
defer fr.Close()
hash := md5.New()
obr := iotest.OneByteReader(fr)
r, err := xz.NewReader(obr, 0)
if err == nil {
b := make([]byte, 1)
var n int
for err == nil {
n, err = r.Read(b)
if n == 1 {
_, _ = hash.Write(b)
}
}
if err == io.EOF {
err = nil
}
}
if err != f.err {
t.Fatalf("%s: wanted error: %v, got: %v\n", f.file, f.err, err)
}
md5sum := fmt.Sprintf("%x", hash.Sum(nil))
if f.md5sum != md5sum {
t.Fatalf(
"%s: wanted md5: %v, got: %v\n", f.file, f.md5sum, md5sum)
}
}()
}
}
func TestBadFiles(t *testing.T) {
testFileList(t, badFiles, false)
}
func TestGoodFiles(t *testing.T) {
testFileList(t, goodFiles, false)
}
func TestUnsupportedFiles(t *testing.T) {
testFileList(t, unsupportedFiles, false)
}
func TestOtherFiles(t *testing.T) {
testFileList(t, otherFiles, false)
}
func TestMemlimit(t *testing.T) {
data, err := readTestFile("words.xz")
if err != nil {
t.Fatal(err)
}
r, err := xz.NewReader(bytes.NewReader(data), 1<<25)
if err == nil {
b := new(bytes.Buffer)
_, err = io.Copy(b, r)
}
if err != xz.ErrMemlimit {
t.Fatalf("wanted error: %v, got: %v\n", xz.ErrMemlimit, err)
}
}
// test to ensure that decoder errors are not returned prematurely
// the test file returns 6 decoded bytes before corruption occurs
func TestPrematureError(t *testing.T) {
data, err := readTestFile("good-2-lzma2-corrupt.xz")
if err != nil {
t.Fatal(err)
}
r, err := xz.NewReader(bytes.NewReader(data), 0)
if err != nil {
t.Fatal(err)
}
b := make([]byte, 2)
n, err := r.Read(b)
if n != 2 || err != nil {
t.Fatalf("Read returned: (%d,%v), expected: (2,%v)\n", n, err, nil)
}
n, err = r.Read(b)
if n != 2 || err != nil {
t.Fatalf("Read returned: (%d,%v), expected: (2,%v)\n", n, err, nil)
}
n, err = r.Read(b)
if n != 2 || err != xz.ErrData {
t.Fatalf("Read returned: (%d,%v), expected: (2,%v)\n",
n, err, xz.ErrData)
}
}
func TestMultipleBadReads(t *testing.T) {
data, err := readTestFile("good-2-lzma2-corrupt.xz")
if err != nil {
t.Fatal(err)
}
r, err := xz.NewReader(bytes.NewReader(data), 0)
if err != nil {
t.Fatal(err)
}
b := make([]byte, 100)
n, err := r.Read(b)
if n != 6 || err != xz.ErrData {
t.Fatalf("Read returned: (%d,%v), expected: (6,%v)\n",
n, err, xz.ErrData)
}
n, err = r.Read(b)
if n != 0 || err != xz.ErrData {
t.Fatalf("Read returned: (%d,%v), expected: (0,%v)\n",
n, err, xz.ErrData)
}
n, err = r.Read(b)
if n != 0 || err != xz.ErrData {
t.Fatalf("Read returned: (%d,%v), expected: (0,%v)\n",
n, err, xz.ErrData)
}
}
// TestByteReads decodes the test files with a one byte input buffer
// and one byte output buffer. This should exercise the stream decoder
// and filter code nicely by testing most of the xzOK exits paths.
func TestByteReads(t *testing.T) {
fileList := badFiles
fileList = append(fileList, goodFiles...)
fileList = append(fileList, unsupportedFiles...)
fileList = append(fileList, otherFiles...)
fileListSmall := []testFile{}
for _, f := range fileList {
if f.file != "zeros-100mb.xz" {
fileListSmall = append(fileListSmall, f)
}
}
testFileListByteReads(t, fileListSmall)
}
func TestMultistream(t *testing.T) {
files := []string{
"good-1-x86-lzma2-offset-2048.xz",
"random-1mb.xz",
"words.xz",
"good-1-x86-lzma2-offset-2048.xz",
"random-1mb.xz",
"words.xz",
}
var readers []io.Reader
for _, f := range files {
data, err := readTestFile(f)
if err != nil {
t.Fatal(err)
}
readers = append(readers, bytes.NewReader(data))
}
mr := io.MultiReader(readers...)
r, err := xz.NewReader(mr, 0)
if err != nil {
t.Fatal(err)
}
for i, f := range files {
r.Multistream(false)
hash := md5.New()
_, err = io.Copy(hash, r)
if err != nil {
t.Fatalf("%s: wanted copy error: %v, got: %v\n", f, nil, err)
}
md5sum := fmt.Sprintf("%x", hash.Sum(nil))
wantedMD5, _ := testFileData(f)
if wantedMD5 != md5sum {
t.Fatalf(
"%s: wanted md5: %v, got: %v\n", f, wantedMD5, md5sum)
}
err = r.Reset(nil)
var wantedErr error
switch {
case i < len(files)-1:
wantedErr = nil
case i == len(files)-1:
wantedErr = io.EOF
}
if wantedErr != err {
t.Fatalf("%s: wanted reset error: %v, got: %v\n",
f, wantedErr, err)
}
}
}
// TestReuseReader decodes the test files reusing the same Reader for
// all files instead of allocating a new Reader for each file.
func TestReuseReader(t *testing.T) {
fileList := badFiles
fileList = append(fileList, goodFiles...)
fileList = append(fileList, unsupportedFiles...)
fileList = append(fileList, otherFiles...)
testFileList(t, fileList, true)
}
// TestReuseReaderPartialReads repeatedly tests decoding a file with a
// reused Reader that has been used immediately before to partially
// decode a file. The amount of partial decoding before the full
// decode is varied on each loop iteration.
func TestReuseReaderPartialReads(t *testing.T) {
data, err := readTestFile("words.xz")
if err != nil {
t.Fatal(err)
}
z, err := xz.NewReader(nil, 0)
if err != nil {
t.Fatal(err)
}
for i := 0; i <= 80000; i += 10000 {
err = z.Reset(bytes.NewReader(data))
if err != nil {
t.Fatal(err)
}
b := make([]byte, i)
_, err = io.ReadFull(z, b)
if err != nil {
t.Fatalf("io.ReadFull: wanted error: %v, got: %v\n", nil, err)
}
err = z.Reset(bytes.NewReader(data))
if err != nil {
t.Fatal(err)
}
hash := md5.New()
_, err = io.Copy(hash, z)
if err != nil {
t.Fatalf("io.Copy: wanted error: %v, got: %v\n", nil, err)
}
md5sum := fmt.Sprintf("%x", hash.Sum(nil))
wantedMD5, _ := testFileData("words.xz")
if wantedMD5 != md5sum {
t.Fatalf(
"hash.Sum: wanted md5: %v, got: %v\n", wantedMD5, md5sum)
}
}
}