diff --git a/cmd/chisel/cmd_debug_check_release_archives.go b/cmd/chisel/cmd_debug_check_release_archives.go index c9c6270e4..9db689805 100644 --- a/cmd/chisel/cmd_debug_check_release_archives.go +++ b/cmd/chisel/cmd_debug_check_release_archives.go @@ -13,7 +13,6 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/cache" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/setup" ) @@ -150,7 +149,7 @@ func computePathObservations(release *setup.Release, archives map[string]archive if err != nil { return nil, err } - dataReader, err := deb.DataReader(pkgReader) + dataReader, err := pkgReader.TarStream() if err != nil { return nil, err } diff --git a/internal/archive/archive.go b/internal/archive/archive.go index d301494b5..a1976c5c0 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -16,11 +16,12 @@ import ( "github.com/canonical/chisel/internal/control" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/pgputil" + "github.com/canonical/chisel/internal/tarball" ) type Archive interface { Options() *Options - Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, error) + Fetch(pkg string) (tarball.PkgReader, *PackageInfo, error) Exists(pkg string) bool Info(pkg string) (*PackageInfo, error) } @@ -140,7 +141,7 @@ func (a *ubuntuArchive) selectPackage(pkg string) (control.Section, *ubuntuIndex return selectedSection, selectedIndex, nil } -func (a *ubuntuArchive) Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, error) { +func (a *ubuntuArchive) Fetch(pkg string) (tarball.PkgReader, *PackageInfo, error) { section, index, err := a.selectPackage(pkg) if err != nil { return nil, nil, err @@ -153,7 +154,7 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, erro return nil, nil, err } info := sectionPackageInfo(section) - return reader, info, nil + return deb.OpenPkg(reader), info, nil } func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) { diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index b89a8d71d..b5bd9a4df 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -1,9 +1,7 @@ package archive_test import ( - "golang.org/x/crypto/openpgp/packet" - . "gopkg.in/check.v1" - + "archive/tar" "crypto/sha256" "crypto/sha512" "debug/elf" @@ -18,6 +16,9 @@ import ( "path/filepath" "strings" + "golang.org/x/crypto/openpgp/packet" + . "gopkg.in/check.v1" + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/archive/testarchive" "github.com/canonical/chisel/internal/tarball" @@ -255,7 +256,7 @@ func (s *httpSuite) TestFetchPackage(c *C) { Name: "mypkg1", Version: "1.1", Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + SHA256: "ff175644a17301e047ac757681f6e16b1c410228d9fe50441bba8438a7c45fa2", }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") @@ -266,7 +267,7 @@ func (s *httpSuite) TestFetchPackage(c *C) { Name: "mypkg4", Version: "1.4", Arch: "amd64", - SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + SHA256: "fe0b0023af4cd5786a2563faadf6ec31e48079a9b176356b008261c1590b6df9", }) c.Assert(read(pkg), Equals, "mypkg4 1.4 data") } @@ -322,13 +323,16 @@ func (s *httpSuite) TestFetchBothDigests(c *C) { Name: "mypkg1", Version: "1.1", Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + SHA256: "ff175644a17301e047ac757681f6e16b1c410228d9fe50441bba8438a7c45fa2", }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") // Pin the cache key: with both digests advertised, the package is cached // under its strongest digest. - sha512Digest := fmt.Sprintf("%x", sha512.Sum512([]byte("mypkg1 1.1 data"))) + sha512Digest := fmt.Sprintf("%x", sha512.Sum512(testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./data", "mypkg1 1.1 data"), + }))) _, err = os.Stat(filepath.Join(options.CacheDir, "sha512", sha512Digest)) c.Assert(err, IsNil) } @@ -359,7 +363,7 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) { Name: "mypkg1", Version: "1.1", Arch: "arm64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + SHA256: "ff175644a17301e047ac757681f6e16b1c410228d9fe50441bba8438a7c45fa2", }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") @@ -370,7 +374,7 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) { Name: "mypkg4", Version: "1.4", Arch: "arm64", - SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + SHA256: "fe0b0023af4cd5786a2563faadf6ec31e48079a9b176356b008261c1590b6df9", }) c.Assert(read(pkg), Equals, "mypkg4 1.4 data") } @@ -382,7 +386,10 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) { err := release.Walk(func(item testarchive.Item) error { if p, ok := item.(*testarchive.Package); ok && p.Name == "mypkg1" { p.Version = fmt.Sprintf("%s.%d", p.Version, i) - p.Data = []byte("package from " + suite) + p.Data = testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./data", "package from "+suite), + }) } return nil }) @@ -410,7 +417,7 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) { Name: "mypkg1", Version: "1.1.2.2", Arch: "amd64", - SHA256: "5448585bdd916e5023eff2bc1bc3b30bcc6ee9db9c03e531375a6a11ddf0913c", + SHA256: "e3732bc52b8a11c8e749266c1eee5548ab02cbaaf84c826277e9ae245d1099b7", }) c.Assert(read(pkg), Equals, "package from jammy-security") @@ -420,7 +427,7 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) { Name: "mypkg2", Version: "1.2", Arch: "amd64", - SHA256: "a4b4f3f3a8fa09b69e3ba23c60a41a1f8144691fd371a2455812572fd02e6f79", + SHA256: "0d229011ec711ef268779580130dc034409e42124d3c25b01e0b71eac90284ad", }) c.Assert(read(pkg), Equals, "mypkg2 1.2 data") } @@ -665,7 +672,7 @@ var packageInfoTests = []struct { Name: "mypkg1", Version: "1.1", Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + SHA256: "ff175644a17301e047ac757681f6e16b1c410228d9fe50441bba8438a7c45fa2", }, }, { summary: "Package not found in archive", @@ -700,12 +707,29 @@ func (s *httpSuite) TestPackageInfo(c *C) { } } -func read(r io.Reader) string { - data, err := io.ReadAll(r) +func read(pkg tarball.PkgReader) string { + tarStream, err := pkg.TarStream() if err != nil { panic(err) } - return string(data) + defer tarStream.Close() + tarReader := tar.NewReader(tarStream) + for { + tarHeader, err := tarReader.Next() + if err == io.EOF { + panic("no data file in package") + } + if err != nil { + panic(err) + } + if tarHeader.Name == "./data" { + data, err := io.ReadAll(tarReader) + if err != nil { + panic(err) + } + return string(data) + } + } } // fetchRequestStatus checks whether a request was made whose URL path diff --git a/internal/archive/testarchive/testarchive.go b/internal/archive/testarchive/testarchive.go index c4ef7822b..af4e1351a 100644 --- a/internal/archive/testarchive/testarchive.go +++ b/internal/archive/testarchive/testarchive.go @@ -106,7 +106,10 @@ func (p *Package) Section() []byte { func (p *Package) Content() []byte { if len(p.Data) == 0 { - return []byte(p.Name + " " + p.Version + " data") + return testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./data", p.Name+" "+p.Version+" data"), + }) } return p.Data } diff --git a/internal/bin/extract.go b/internal/bin/extract.go new file mode 100644 index 000000000..fc5508275 --- /dev/null +++ b/internal/bin/extract.go @@ -0,0 +1,31 @@ +package bin + +import ( + "io" + + "github.com/ulikunitz/xz" +) + +type Pkg struct { + reader io.ReadSeekCloser +} + +func OpenPkg(reader io.ReadSeekCloser) *Pkg { + return &Pkg{reader: reader} +} + +func (p *Pkg) TarStream() (io.ReadCloser, error) { + _, err := p.reader.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + xzReader, err := xz.NewReader(p.reader) + if err != nil { + return nil, err + } + return io.NopCloser(xzReader), nil +} + +func (p *Pkg) Close() error { + return p.reader.Close() +} diff --git a/internal/bin/extract_test.go b/internal/bin/extract_test.go new file mode 100644 index 000000000..4d60358d0 --- /dev/null +++ b/internal/bin/extract_test.go @@ -0,0 +1,38 @@ +package bin_test + +import ( + "archive/tar" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/bin" + "github.com/canonical/chisel/internal/tarball" + "github.com/canonical/chisel/internal/testutil" +) + +var _ tarball.PkgReader = (*bin.Pkg)(nil) + +func (s *S) TestPkgTarStream(c *C) { + pkg := testutil.NewBinPkg(testutil.MustMakeBin([]testutil.TarEntry{ + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "content"), + })) + + // Each call returns a fresh stream over the same content. + for range 2 { + tarStream, err := pkg.TarStream() + c.Assert(err, IsNil) + tarReader := tar.NewReader(tarStream) + _, err = tarReader.Next() + c.Assert(err, IsNil) + err = tarStream.Close() + c.Assert(err, IsNil) + } +} + +func (s *S) TestPkgTarStreamInvalid(c *C) { + pkg := testutil.NewBinPkg([]byte("not an xz stream")) + + _, err := pkg.TarStream() + c.Assert(err, ErrorMatches, "xz.*") +} diff --git a/internal/bin/log.go b/internal/bin/log.go new file mode 100644 index 000000000..e0f72a8df --- /dev/null +++ b/internal/bin/log.go @@ -0,0 +1,53 @@ +package bin + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/bin/suite_test.go b/internal/bin/suite_test.go new file mode 100644 index 000000000..0a450d8ec --- /dev/null +++ b/internal/bin/suite_test.go @@ -0,0 +1,25 @@ +package bin_test + +import ( + "testing" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/bin" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + bin.SetDebug(true) + bin.SetLogger(c) +} + +func (s *S) TearDownTest(c *C) { + bin.SetDebug(false) + bin.SetLogger(nil) +} diff --git a/internal/deb/extract.go b/internal/deb/extract.go index bf543c2d8..4b56e12e5 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -10,10 +10,22 @@ import ( "github.com/ulikunitz/xz" ) -// DataReader takes a Reader for the ar file belonging to a Debian package and -// returns a Reader to the inner tarball. -func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { - arReader := ar.NewReader(pkgReader) +type Pkg struct { + reader io.ReadSeekCloser +} + +func OpenPkg(reader io.ReadSeekCloser) *Pkg { + return &Pkg{reader: reader} +} + +// TarStream returns a ReadCloser to the inner tarball of +// a Debian package. +func (p *Pkg) TarStream() (io.ReadCloser, error) { + _, err := p.reader.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + arReader := ar.NewReader(p.reader) var dataReader io.ReadCloser for dataReader == nil { arHeader, err := arReader.Next() @@ -47,3 +59,7 @@ func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { return dataReader, nil } + +func (p *Pkg) Close() error { + return p.reader.Close() +} diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go new file mode 100644 index 000000000..73f8d245f --- /dev/null +++ b/internal/deb/extract_test.go @@ -0,0 +1,52 @@ +package deb_test + +import ( + "archive/tar" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/tarball" + "github.com/canonical/chisel/internal/testutil" +) + +var _ tarball.PkgReader = (*deb.Pkg)(nil) + +func (s *S) TestPkgTarStream(c *C) { + pkg := testutil.NewDebPkg(testutil.PackageData["test-package"]) + + // Each call returns a fresh stream over the same content. + for i := 0; i < 2; i++ { + tarStream, err := pkg.TarStream() + c.Assert(err, IsNil) + tarReader := tar.NewReader(tarStream) + _, err = tarReader.Next() + c.Assert(err, IsNil) + err = tarStream.Close() + c.Assert(err, IsNil) + } +} + +func (s *S) TestPkgExtract(c *C) { + pkg := testutil.NewDebPkg(testutil.PackageData["test-package"]) + + dir := c.MkDir() + err := tarball.Extract(pkg, &tarball.ExtractOptions{ + Package: "test-package", + TargetDir: dir, + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": {{Path: "/dir/file"}}, + "/dir/nested/": {{ + Path: "/dir/nested/", + }}, + }, + }) + c.Assert(err, IsNil) + + result := testutil.TreeDump(dir) + c.Assert(result, DeepEquals, map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + "/dir/nested/": "dir 0755", + }) +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index ce7911621..3253093af 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -146,7 +146,7 @@ func Run(options *RunOptions) error { } // Fetch all packages, using the selection order. - packages := make(map[string]io.ReadSeekCloser) + packages := make(map[string]tarball.PkgReader) var pkgInfos []manifestutil.PackageInfo for _, slice := range options.Selection.Slices { if packages[slice.Package] != nil { diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 40db2cbac..4dec52afd 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -12,11 +12,17 @@ import ( "strings" "syscall" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/strdist" ) +type PkgReader interface { + // TarStream returns a reader over the raw, unparsed tar stream. + // Each call returns a fresh stream, from its start. + TarStream() (io.ReadCloser, error) + io.Closer +} + type ExtractOptions struct { Package string TargetDir string @@ -58,7 +64,7 @@ func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { return options, nil } -func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { +func Extract(pkg PkgReader, options *ExtractOptions) (err error) { defer func() { if err != nil { err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) @@ -79,15 +85,15 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { return err } - return extractData(pkgReader, validOpts) + return extractEntries(pkg, validOpts) } -func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := deb.DataReader(pkgReader) +func extractEntries(pkg PkgReader, options *ExtractOptions) error { + tarStream, err := pkg.TarStream() if err != nil { return err } - defer dataReader.Close() + defer tarStream.Close() oldUmask := syscall.Umask(0) defer func() { @@ -117,10 +123,10 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { // create them with the permissions defined in the tarball. // // The assumption is that the tar entries of the parent directories appear - // before the entry for the file itself. This is the case for .deb files but - // not for all tarballs. + // before the entry for the file itself. This is the case for the tarballs + // produced by common packaging tools but not for all tarballs. tarDirMode := make(map[string]fs.FileMode) - tarReader := tar.NewReader(dataReader) + tarReader := tar.NewReader(tarStream) for { tarHeader, err := tarReader.Next() if err == io.EOF { @@ -261,11 +267,7 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { ExtractOptions: options, pendingLinks: pendingHardLinks, } - _, err := pkgReader.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = extractHardLinks(pkgReader, extractHardLinkOptions) + err = extractHardLinks(pkg, extractHardLinkOptions) if err != nil { return err } @@ -299,14 +301,14 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. -func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := deb.DataReader(pkgReader) +func extractHardLinks(pkg PkgReader, opts *extractHardLinkOptions) error { + tarStream, err := pkg.TarStream() if err != nil { return err } - defer dataReader.Close() + defer tarStream.Close() - tarReader := tar.NewReader(dataReader) + tarReader := tar.NewReader(tarStream) for { tarHeader, err := tarReader.Next() if err == io.EOF { diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index fd5eb6147..93b5ff15c 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -1,7 +1,6 @@ package tarball_test import ( - "bytes" "os" "path" "path/filepath" @@ -17,7 +16,7 @@ import ( type extractTest struct { summary string - pkgdata []byte + pkg *testutil.TestPkg options tarball.ExtractOptions hackopt func(c *C, o *tarball.ExtractOptions) result map[string]string @@ -28,14 +27,14 @@ type extractTest struct { var extractTests = []extractTest{{ summary: "Extract nothing", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: nil, }, result: map[string]string{}, }, { summary: "Extract a few entries", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/file": []tarball.ExtractInfo{{ @@ -69,7 +68,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Extract a few entries, nil Create closure", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/file": []tarball.ExtractInfo{{ @@ -105,7 +104,7 @@ var extractTests = []extractTest{{ }, }, { summary: "Copy a couple of entries elsewhere", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/file": []tarball.ExtractInfo{{ @@ -127,7 +126,7 @@ var extractTests = []extractTest{{ notCreated: []string{"/foo/", "/foo/bar/"}, }, { summary: "Copy same file twice", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/file": []tarball.ExtractInfo{{ @@ -147,7 +146,7 @@ var extractTests = []extractTest{{ notCreated: []string{"/dir/bar/", "/dir/foo/"}, }, { summary: "Globbing a single dir level", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/s*/": []tarball.ExtractInfo{{ @@ -162,7 +161,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Globbing for files with multiple levels at once", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/s**": []tarball.ExtractInfo{{ @@ -180,7 +179,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Globbing multiple paths", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/s**": []tarball.ExtractInfo{{ @@ -202,7 +201,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Globbing must have matching source and target", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/foo/b**": []tarball.ExtractInfo{{ @@ -213,7 +212,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /foo/b\*\*`, }, { summary: "Globbing must also have a single target", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/foo/b**": []tarball.ExtractInfo{{ @@ -226,7 +225,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /foo/b\*\*`, }, { summary: "Globbing cannot change modes", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/n**": []tarball.ExtractInfo{{ @@ -238,7 +237,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /dir/n\*\*`, }, { summary: "Missing file", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/missing-file": []tarball.ExtractInfo{{ @@ -249,7 +248,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at /missing-file`, }, { summary: "Missing directory", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/missing-dir/": []tarball.ExtractInfo{{ @@ -260,7 +259,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at /missing-dir/`, }, { summary: "Missing glob", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/missing-dir/**": []tarball.ExtractInfo{{ @@ -271,7 +270,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at /missing-dir/\*\*`, }, { summary: "Missing multiple entries", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/missing-file": []tarball.ExtractInfo{{ @@ -285,7 +284,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at:\n- /missing-dir/\n- /missing-file`, }, { summary: "Optional entries may be missing", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/": []tarball.ExtractInfo{{ @@ -307,7 +306,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Optional entries mixed in cannot be missing", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/missing-file": []tarball.ExtractInfo{{ @@ -322,11 +321,11 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at /dir/missing-file`, }, { summary: "Extract non-ASCII path and preserve parent directories permissions", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Dir(0766, "./日本/"), testutil.Reg(0644, "./日本/語", "whatever"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/日本/語": []tarball.ExtractInfo{{ @@ -341,7 +340,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Entries for same destination must have the same mode", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/": []tarball.ExtractInfo{{ @@ -356,11 +355,11 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": path /dir/ requested twice with diverging mode: 0777 != 0000`, }, { summary: "Single hard link entry can be extracted with the content entry", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Reg(0644, "./file", "text for file"), testutil.Hrd(0644, "./hardlink", "./file"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -375,11 +374,11 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Single hard link entry can be extracted without the content entry", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Reg(0644, "./file", "text for file"), testutil.Hrd(0644, "./hardlink", "./file"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/hardlink": []tarball.ExtractInfo{{ @@ -393,10 +392,10 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Dangling hard link", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink", "./non-existing-target"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/hardlink": []tarball.ExtractInfo{{ @@ -407,11 +406,11 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": cannot create hard link /hardlink: no content at /non-existing-target`, }, { summary: "Multiple dangling hard links", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink1", "./non-existing-target"), testutil.Hrd(0644, "./hardlink2", "./non-existing-target"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -422,11 +421,11 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": cannot create hard link /hardlink1: no content at /non-existing-target`, }, { summary: "Hard link does not follow the symlink", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Lnk(0644, "./symlink", "./file"), testutil.Hrd(0644, "./hardlink", "./symlink"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -441,7 +440,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Explicit extraction overrides existing file", - pkgdata: testutil.PackageData["test-package"], + pkg: testutil.NewTestPkg(testutil.TestPackageEntries...), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/": []tarball.ExtractInfo{{ @@ -460,10 +459,10 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Hardlink cannot escape target directory", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink", "/etc/group"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -474,10 +473,10 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": invalid link target /etc/group`, }, { summary: "Cannot extract outside of target directory", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Reg(0644, "./../file", "hijacking system file"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -511,7 +510,7 @@ func (s *S) TestExtract(c *C) { test.hackopt(c, &options) } - err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) + err := tarball.Extract(test.pkg, &options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -538,16 +537,16 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string - pkgdata []byte + pkg *testutil.TestPkg options tarball.ExtractOptions calls map[string][]tarball.ExtractInfo }{{ summary: "Create is called with the set of ExtractInfo(s) that match the file", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Dir(0766, "./dir/"), testutil.Reg(0644, "./dir/file", "whatever"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/": []tarball.ExtractInfo{{ @@ -622,7 +621,7 @@ func (s *S) TestExtractCreateCallback(c *C) { return nil } - err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) + err := tarball.Extract(test.pkg, &options) c.Assert(err, IsNil) c.Assert(createExtractInfos, DeepEquals, test.calls) diff --git a/internal/testutil/archive.go b/internal/testutil/archive.go index d06fd1b0c..260dd608e 100644 --- a/internal/testutil/archive.go +++ b/internal/testutil/archive.go @@ -3,9 +3,10 @@ package testutil import ( "bytes" "fmt" - "io" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/tarball" ) type TestArchive struct { @@ -26,7 +27,7 @@ func (a *TestArchive) Options() *archive.Options { return &a.Opts } -func (a *TestArchive) Fetch(pkgName string) (io.ReadSeekCloser, *archive.PackageInfo, error) { +func (a *TestArchive) Fetch(pkgName string) (tarball.PkgReader, *archive.PackageInfo, error) { pkg, ok := a.Packages[pkgName] if !ok { return nil, nil, fmt.Errorf("cannot find package %q in archive", pkgName) @@ -37,7 +38,7 @@ func (a *TestArchive) Fetch(pkgName string) (io.ReadSeekCloser, *archive.Package SHA256: pkg.Hash, Arch: pkg.Arch, } - return ReadSeekNopCloser(bytes.NewReader(pkg.Data)), info, nil + return deb.OpenPkg(ReadSeekNopCloser(bytes.NewReader(pkg.Data))), info, nil } func (a *TestArchive) Exists(pkg string) bool { diff --git a/internal/testutil/pkgdata.go b/internal/testutil/pkgdata.go index f901873d3..5eca2e8bb 100644 --- a/internal/testutil/pkgdata.go +++ b/internal/testutil/pkgdata.go @@ -8,6 +8,7 @@ import ( "github.com/blakesmith/ar" "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" ) var PackageData = map[string][]byte{} @@ -160,6 +161,37 @@ func MustMakeDeb(entries []TarEntry) []byte { return data } +func compressBytesXZ(input []byte) ([]byte, error) { + var buf bytes.Buffer + writer, err := xz.NewWriter(&buf) + if err != nil { + return nil, err + } + if _, err = writer.Write(input); err != nil { + return nil, err + } + if err = writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func MakeBin(entries []TarEntry) ([]byte, error) { + tarData, err := makeTar(entries) + if err != nil { + return nil, err + } + return compressBytesXZ(tarData) +} + +func MustMakeBin(entries []TarEntry) []byte { + data, err := MakeBin(entries) + if err != nil { + panic(err) + } + return data +} + // Reg is a shortcut for creating a regular file TarEntry structure (with // tar.Typeflag set tar.TypeReg). Reg stands for "REGular file". func Reg(mode int64, path, content string) TarEntry { diff --git a/internal/testutil/pkgreader.go b/internal/testutil/pkgreader.go new file mode 100644 index 000000000..3ecb4ac2a --- /dev/null +++ b/internal/testutil/pkgreader.go @@ -0,0 +1,41 @@ +package testutil + +import ( + "bytes" + "io" + + "github.com/canonical/chisel/internal/bin" + "github.com/canonical/chisel/internal/deb" +) + +// TestPkg is a PkgReader over an in-memory tar stream. +type TestPkg struct { + tarData []byte +} + +// NewTestPkg returns a TestPkg holding a tar stream with the given entries. +func NewTestPkg(entries ...TarEntry) *TestPkg { + data, err := makeTar(entries) + if err != nil { + panic(err) + } + return &TestPkg{tarData: data} +} + +func (p *TestPkg) TarStream() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(p.tarData)), nil +} + +func (p *TestPkg) Close() error { + return nil +} + +// NewDebPkg returns a deb.Pkg over the given Debian package data. +func NewDebPkg(data []byte) *deb.Pkg { + return deb.OpenPkg(ReadSeekNopCloser(bytes.NewReader(data))) +} + +// NewBinPkg returns a bin.Pkg over the given bin package data. +func NewBinPkg(data []byte) *bin.Pkg { + return bin.OpenPkg(ReadSeekNopCloser(bytes.NewReader(data))) +}