From 86f032c54c13ad1210406643c55f7bd59345cf3e Mon Sep 17 00:00:00 2001 From: Marcus Aram <4435760+marcusaram@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:12:22 +0100 Subject: [PATCH 1/8] Maker: write Armbian's ext4 root in Go, without e2fsprogs The new ext4 package changes an ext4 file system without mounting it: write, replace and remove files, make directories and symlinks, for the features Armbian's root has (64bit, extents, flex_bg, metadata_csum with a seed, dir_index). It works in whole blocks, keeps every change in memory until Flush, and refuses a file system that is not clean or has a feature it does not know. Squirrel Maker uses it instead of debugfs and e2fsck, so the e2fsprogs build for macOS, its GPL binaries in the app and the authopen file descriptor trick are gone, and the same code can run on Windows. Tested in make test-linux: a file system made like Armbian's, with an indexed directory, changed and judged by e2fsck -fn and read back by the Linux kernel; and, opt-in, the real Armbian image made into a card and updated, judged by e2fsck. --- .github/workflows/release.yml | 9 +- .gitignore | 1 - Makefile | 46 +- README.md | 4 +- THIRD_PARTY_NOTICES.md | 10 +- internal/ext4/ext4.go | 576 +++++++++++++++++++ internal/ext4/ext4_test.go | 216 +++++++ internal/ext4/write.go | 880 +++++++++++++++++++++++++++++ internal/maker/armbian.go | 170 +++--- internal/maker/armbian_test.go | 35 +- internal/maker/update.go | 49 +- internal/maker/update_darwin.go | 18 +- internal/maker/update_other.go | 2 +- internal/maker/update_test.go | 35 +- internal/makerui/makerui.go | 30 +- internal/makerui/payload/README.md | 5 +- 16 files changed, 1844 insertions(+), 242 deletions(-) create mode 100644 internal/ext4/ext4.go create mode 100644 internal/ext4/ext4_test.go create mode 100644 internal/ext4/write.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1b8d0ce..13a941f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: - name: squirreld archives run: make release VERSION="$VERSION" - name: Packages for a new card - run: make payload-linux VERSION="$VERSION" + run: make payload VERSION="$VERSION" - uses: actions/upload-artifact@v4 with: name: squirreld @@ -38,14 +38,14 @@ jobs: if-no-files-found: error - uses: actions/upload-artifact@v4 with: - name: payload-linux + name: payload path: | internal/makerui/payload/files internal/makerui/payload/files-armv6 internal/makerui/payload/armbian if-no-files-found: error - # What needs a Mac: e2fsprogs for the ROCK 4C+ card, and the app. + # What needs a Mac: the app. macos: needs: linux runs-on: macos-latest @@ -56,13 +56,12 @@ jobs: go-version-file: go.mod - uses: actions/download-artifact@v4 with: - name: payload-linux + name: payload path: internal/makerui/payload - name: Version from the tag run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_ENV" - name: Squirrel Maker run: | - make payload-mac VERSION="$VERSION" make app-bundle maker-bin VERSION="$VERSION" cd dist && zip -q "squirrel-maker-$VERSION-macos.zip" squirrel-maker - uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 1174bbd..9184f85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,4 @@ .DS_Store /.claude/ /internal/makerui/payload/armbian/ -/internal/makerui/payload/mac/ /internal/makerui/payload/files-armv6/ diff --git a/Makefile b/Makefile index 348a324..6677942 100644 --- a/Makefile +++ b/Makefile @@ -34,9 +34,9 @@ test: .PHONY: test-linux test-linux: mkdir -p dist/test - for p in fatfs preserve gadget archive driveimage nbd; do GOOS=linux go test -c -o dist/test/$$p.test ./internal/$$p; done + for p in fatfs preserve gadget archive driveimage nbd ext4; do GOOS=linux go test -c -o dist/test/$$p.test ./internal/$$p; done docker run --rm --privileged -v "$(CURDIR):/src:ro" debian:trixie-slim sh -c '\ - apt-get update -qq >/dev/null && apt-get install -y -qq xfsprogs exfatprogs >/dev/null 2>&1; \ + apt-get update -qq >/dev/null && apt-get install -y -qq xfsprogs exfatprogs e2fsprogs >/dev/null 2>&1; \ truncate -s 1G /x.img && mkfs.xfs -q /x.img && mkdir /xfs && mount -o loop /x.img /xfs && \ cd /src/internal/fatfs && /src/dist/test/fatfs.test && \ cd /src/internal/gadget && /src/dist/test/gadget.test -test.v -test.run Mounted && \ @@ -44,7 +44,8 @@ test-linux: cd /src/internal/driveimage && /src/dist/test/driveimage.test -test.v && \ cd /src/internal/preserve && TBX_TEST_DIR=/xfs TBX_EXPECT_REFLINK=1 /src/dist/test/preserve.test -test.v && \ for i in 0 1 2 3; do [ -e /dev/nbd$$i ] || mknod /dev/nbd$$i b 43 $$((i*32)); done && \ - cd /src/internal/nbd && /src/dist/test/nbd.test -test.v' + cd /src/internal/nbd && /src/dist/test/nbd.test -test.v && \ + cd /src/internal/ext4 && /src/dist/test/ext4.test -test.v' # Validates the generated cloud-init files against a real Raspberry Pi OS # image (uncompressed .img), e.g. PIOS_IMG=~/Downloads/2026-10-06-raspios-trixie-arm64-lite.img @@ -72,8 +73,6 @@ testimages: # xfsprogs, exfatprogs and their dependencies, worked out against the # package list of the pinned Armbian image (deploy/armbian/dpkg-status), # the USB overlay and the unit. -# - mac/: debugfs and e2fsck, which the app uses to put files on -# Armbian's ext4 root; built from e2fsprogs for Apple silicon and Intel. PAYLOAD := internal/makerui/payload XFS_DEBS := xfsprogs libinih1 liburcu8t64 python3-dbus # download fetches packages into the current directory in a container, @@ -86,16 +85,11 @@ download = ok=; for i in 1 2 3; do rm -f ./*.deb; apt-get update -qq >/dev/null for p in $(foreach p,$(1),$(firstword $(subst :, ,$(p)))); do \ ls "$${p}"_*.deb >/dev/null 2>&1 || { echo "$${p} missing" >&2; exit 1; }; done; ls ARMBIAN_PKGS := network-manager dnsmasq-base xfsprogs exfatprogs rfkill -E2FS_VERSION := 1.47.4 -E2FS_SHA256 := fd5bf388cbdbe006a3d3b318d983b2948382440acc85a87f1e7d108653e8db0b -E2FS := dist/e2fsprogs-$(E2FS_VERSION) -# payload is built in two parts: what needs Docker (payload-linux) and what -# needs a Mac (payload-mac). CI builds them on different machines. -.PHONY: payload payload-linux payload-mac -payload: payload-linux payload-mac - -payload-linux: dist/squirreld-linux-arm64 dist/squirreld-linux-armv6 +# payload needs Docker; CI builds it on Linux and the Mac app around it on +# a Mac. +.PHONY: payload +payload: dist/squirreld-linux-arm64 dist/squirreld-linux-armv6 rm -rf $(PAYLOAD)/files $(PAYLOAD)/files-armv6 $(PAYLOAD)/armbian mkdir -p $(PAYLOAD)/files/debs $(PAYLOAD)/files-armv6/debs $(PAYLOAD)/armbian/debs cp dist/squirreld-linux-arm64 $(PAYLOAD)/files/squirreld @@ -118,28 +112,6 @@ payload-linux: dist/squirreld-linux-arm64 dist/squirreld-linux-armv6 apt-get install -qq -y --download-only --no-install-recommends -o Dir::Cache::archives=/tmp/a $(ARMBIAN_PKGS) >/dev/null; \ cp /tmp/a/*.deb /out/ && ls /out' -payload-mac: $(E2FS)/debugfs - rm -rf $(PAYLOAD)/mac && mkdir -p $(PAYLOAD)/mac - cp $(E2FS)/debugfs $(E2FS)/e2fsck $(PAYLOAD)/mac/ - -# One rule for both tools (with two targets, make -j would run it twice -# at once); e2fsck is made alongside debugfs. -$(E2FS)/debugfs: - rm -rf $(E2FS) && mkdir -p $(E2FS) - curl -sfL -o $(E2FS)/src.tar.xz https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v$(E2FS_VERSION)/e2fsprogs-$(E2FS_VERSION).tar.xz - echo "$(E2FS_SHA256) $(E2FS)/src.tar.xz" | shasum -a 256 -c - - tar xf $(E2FS)/src.tar.xz -C $(E2FS) - for a in arm64 x86_64; do \ - mkdir -p $(E2FS)/build-$$a && cd $(E2FS)/build-$$a && \ - CFLAGS="-arch $$a -O2 -mmacosx-version-min=13.0" LDFLAGS="-arch $$a -mmacosx-version-min=13.0" \ - ../e2fsprogs-$(E2FS_VERSION)/configure --host=$$a-apple-darwin --disable-nls --disable-fuse2fs \ - --disable-defrag --disable-imager --disable-e2initrd-helper --disable-uuidd > configure.log && \ - $(MAKE) -s libs > make.log && $(MAKE) -s -C debugfs debugfs >> make.log && $(MAKE) -s -C e2fsck e2fsck >> make.log && \ - cd $(CURDIR) || exit 1; \ - done - lipo -create $(E2FS)/build-arm64/debugfs/debugfs $(E2FS)/build-x86_64/debugfs/debugfs -output $(E2FS)/debugfs - lipo -create $(E2FS)/build-arm64/e2fsck/e2fsck $(E2FS)/build-x86_64/e2fsck/e2fsck -output $(E2FS)/e2fsck - # Squirrel Maker, the Mac app: its own window, Dock icon and menu, built # with Wails (cgo, so Xcode's command line tools). One binary for Apple # silicon and Intel. @@ -176,6 +148,6 @@ maker-bin: .PHONY: clean clean: - rm -rf dist $(PAYLOAD)/files $(PAYLOAD)/files-armv6 $(PAYLOAD)/armbian $(PAYLOAD)/mac + rm -rf dist $(PAYLOAD)/files $(PAYLOAD)/files-armv6 $(PAYLOAD)/armbian FORCE: diff --git a/README.md b/README.md index 0b25ceb..baa0725 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ The original Zero W gets 32-bit Raspberry Pi OS with an ARMv6 build of squirreld ### Radxa ROCK 4C+ -The ROCK 4C+ gets [Armbian](https://www.armbian.com/) (Debian trixie, minimal). There is no stable Armbian release for this board, only community builds; Squirrel Maker uses one that was checked. Armbian keeps everything on ext4, which macOS cannot write, so Squirrel Maker writes its files with `debugfs` (and checks with `e2fsck`) and asks for your password. At the end of the first start the board restarts once; the car then sees the drive. Ethernet stays with Armbian's own network setup; NetworkManager handles wifi and the hotspot. The USB device mode follows the [teslausb wiki](https://github.com/marcone/teslausb/wiki/Rock-Pi-4C-plus-Installation). +The ROCK 4C+ gets [Armbian](https://www.armbian.com/) (Debian trixie, minimal). There is no stable Armbian release for this board, only community builds; Squirrel Maker uses one that was checked. Armbian keeps everything on ext4, which macOS and Windows cannot write, so Squirrel Maker writes its files itself, without mounting the card: it only touches a file system that is clean and uses ext4 features it knows, and reads everything back afterwards. Updating such a card asks for your password. At the end of the first start the board restarts once; the car then sees the drive. Ethernet stays with Armbian's own network setup; NetworkManager handles wifi and the hotspot. The USB device mode follows the [teslausb wiki](https://github.com/marcone/teslausb/wiki/Rock-Pi-4C-plus-Installation). ### Clips on a USB drive @@ -229,7 +229,7 @@ make app # dist/Squirrel Maker.app and its zip (Apple silicon and Intel make maker # dist/squirrel-maker, the command line version ``` -`make app` and `make maker` build in two parts, which CI runs on different machines: `make payload-linux` (squirreld and the packages for the card, with Docker) and `make payload-mac` (see below), then `make app-bundle` or `make maker-bin` around them. They also build `debugfs` and `e2fsck` from the [e2fsprogs](https://e2fsprogs.sourceforge.net/) source (checksum checked) and fetch the packages a new card installs at its first start, so that the first start needs no internet. +`make app` and `make maker` build in two steps, which CI runs on different machines: `make payload` (squirreld, and the packages a new card installs at its first start, fetched with Docker so that the first start needs no internet), then `make app-bundle` or `make maker-bin` around it. To run squirreld on your computer against a demo image with fake footage from six cameras: diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index ade3356..0924c92 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -36,15 +36,11 @@ squirreld with your version. The library's source and license text: | [github.com/ulikunitz/xz](https://github.com/ulikunitz/xz) | BSD-3-Clause, © 2014-2022 Ulrich Kunitz | | [golang.org/x/sys](https://pkg.go.dev/golang.org/x/sys) | BSD-3-Clause, © 2009 The Go Authors | -## Programs and packages the Mac app carries +## Packages Squirrel Maker carries -These are separate programs, put on the SD card or run as they are; they are -not part of Squirrel and keep their own licenses. +These are separate programs, put on the SD card as they are; they are not +part of Squirrel and keep their own licenses. -- **e2fsprogs 1.47.4** (`debugfs`, `e2fsck`, to write to the ext4 card of a - Radxa ROCK 4C+): GPL-2.0. Built unchanged from the release at - - (the `Makefile` checks its SHA-256). That tarball is the complete source. - **Debian packages** for a new card, put on it unchanged and installed at its first start: `xfsprogs`, `libinih1`, `liburcu8t64` and `python3-dbus` for Raspberry Pi OS (from Debian trixie for 64-bit, from Raspbian trixie for the diff --git a/internal/ext4/ext4.go b/internal/ext4/ext4.go new file mode 100644 index 0000000..9664bea --- /dev/null +++ b/internal/ext4/ext4.go @@ -0,0 +1,576 @@ +// Package ext4 reads and changes an ext4 file system in an image or on a +// card without mounting it: just what Squirrel Maker needs to put its +// files on an Armbian root (write, replace and remove files, make +// directories and symlinks), for the features that root has. +// +// It reads and writes whole blocks only, which raw disks require, and +// keeps every change in memory until Flush, so an error halfway leaves +// the file system as it was. It refuses a file system it does not fully +// understand (an unknown feature) or that is not clean (a journal to +// replay, errors, orphans), rather than guess. +package ext4 + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "io/fs" + "path" + "sort" + "strings" + "time" +) + +// Device is what the file system lives on: an image file or a card. +type Device interface { + io.ReaderAt + io.WriterAt +} + +var ( + ErrNotExist = fs.ErrNotExist + ErrUnsupported = errors.New("ext4: unsupported file system") + ErrCorrupt = errors.New("ext4: corrupt file system") + ErrNoSpace = errors.New("ext4: no space left") +) + +const ( + compatHasJournal = 0x4 + compatExtAttr = 0x8 + compatResizeInode = 0x10 + compatDirIndex = 0x20 + + incompatFiletype = 0x2 + incompatRecover = 0x4 + incompatExtents = 0x40 + incompat64bit = 0x80 + incompatFlexBG = 0x200 + incompatCsumSeed = 0x2000 + + roSparseSuper = 0x1 + roLargeFile = 0x2 + roHugeFile = 0x8 + roDirNlink = 0x20 + roExtraIsize = 0x40 + roMetadataCsum = 0x400 + + knownCompat = compatHasJournal | compatExtAttr | compatResizeInode | compatDirIndex + knownIncompat = incompatFiletype | incompatExtents | incompat64bit | incompatFlexBG | incompatCsumSeed + knownRO = roSparseSuper | roLargeFile | roHugeFile | roDirNlink | roExtraIsize | roMetadataCsum + + stateValid = 0x1 + stateErrors = 0x2 + + bgInodeUninit = 0x1 + bgBlockUninit = 0x2 + + inodeIndexFl = 0x1000 + inodeExtentsFl = 0x80000 + inodeHugeFl = 0x40000 + + extentMagic = 0xF30A + rootIno = 2 + + modeDir = 0o040000 + modeRegular = 0o100000 + modeSymlink = 0o120000 + modeType = 0o170000 +) + +var castagnoli = crc32.MakeTable(crc32.Castagnoli) + +// crc is crc32c as the kernel computes it: no inversion before or after. +func crc(seed uint32, b []byte) uint32 { return ^crc32.Update(^seed, castagnoli, b) } + +func le16(b []byte) uint16 { return binary.LittleEndian.Uint16(b) } +func le32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) } +func put16(b []byte, v uint16) { binary.LittleEndian.PutUint16(b, v) } +func put32(b []byte, v uint32) { binary.LittleEndian.PutUint32(b, v) } + +// FS is an opened ext4 file system. +type FS struct { + dev Device + offset int64 // where the file system starts on dev + + bs int64 // block size + blocks uint64 + firstData uint64 + perGroup uint32 // blocks per group + inodesPer uint32 + inodeSize int + descSize int + groups uint32 + firstIno uint32 + csum bool // metadata_csum + csumSeed uint32 // for every checksum but the superblock's + wantExtra uint16 // i_extra_isize for new inodes + sbBlock uint64 // the block that holds the superblock + sbOff int // where in that block + gdtBlock uint64 // first block of the group descriptors + cache map[uint64][]byte + dirty map[uint64]bool + touched map[uint32]bool // groups whose counts or bitmaps changed + now uint32 // the time stamped on what changes +} + +// Open reads the superblock at offset on dev and checks that the file +// system is one this package can change safely. +func Open(dev Device, offset int64) (*FS, error) { + f := &FS{dev: dev, offset: offset, bs: 1024, cache: map[uint64][]byte{}, dirty: map[uint64]bool{}, + now: uint32(time.Now().Unix())} + // The superblock is 1024 bytes in; read the 4096 bytes around it + // (whole sectors) before the block size is known. + head := make([]byte, 4096) + if _, err := dev.ReadAt(head, offset); err != nil { + return nil, fmt.Errorf("ext4: reading the superblock: %w", err) + } + sb := head[1024:2048] + if le16(sb[0x38:]) != 0xEF53 { + return nil, fmt.Errorf("%w: no ext4 superblock", ErrUnsupported) + } + logBS := le32(sb[0x18:]) + if logBS > 6 { + return nil, fmt.Errorf("%w: block size", ErrCorrupt) + } + f.bs = 1024 << logBS + compat, incompat, ro := le32(sb[0x5C:]), le32(sb[0x60:]), le32(sb[0x64:]) + switch { + case compat&^knownCompat != 0 || incompat&^knownIncompat != 0 || ro&^knownRO != 0: + return nil, fmt.Errorf("%w: features %#x/%#x/%#x beyond %#x/%#x/%#x", ErrUnsupported, + compat, incompat, ro, knownCompat, knownIncompat, knownRO) + case incompat&incompatFiletype == 0 || incompat&incompatExtents == 0: + return nil, fmt.Errorf("%w: needs filetype and extents", ErrUnsupported) + case le32(sb[0x1C:]) != logBS: + return nil, fmt.Errorf("%w: bigalloc", ErrUnsupported) + } + f.firstData = uint64(le32(sb[0x14:])) + f.perGroup = le32(sb[0x20:]) + f.inodesPer = le32(sb[0x28:]) + f.inodeSize = int(le16(sb[0x58:])) + f.firstIno = le32(sb[0x54:]) + f.blocks = uint64(le32(sb[0x4:])) + f.descSize = 32 + if incompat&incompat64bit != 0 { + f.blocks |= uint64(le32(sb[0x150:])) << 32 + f.descSize = int(le16(sb[0xFE:])) + } + if f.perGroup == 0 || f.inodesPer == 0 || f.inodeSize < 128 || f.descSize < 32 || f.inodesPer%8 != 0 { + return nil, fmt.Errorf("%w: geometry", ErrCorrupt) + } + f.groups = uint32((f.blocks - f.firstData + uint64(f.perGroup) - 1) / uint64(f.perGroup)) + f.csum = ro&roMetadataCsum != 0 + if f.csum { + if sb[0x175] != 1 { + return nil, fmt.Errorf("%w: checksum type %d", ErrUnsupported, sb[0x175]) + } + if crc(^uint32(0), sb[:0x3FC]) != le32(sb[0x3FC:]) { + return nil, fmt.Errorf("%w: superblock checksum", ErrCorrupt) + } + if incompat&incompatCsumSeed != 0 { + f.csumSeed = le32(sb[0x270:]) + } else { + f.csumSeed = crc(^uint32(0), sb[0x68:0x78]) + } + } + f.wantExtra = 32 + if ro&roExtraIsize != 0 { + if w := le16(sb[0x15E:]); w >= 4 { + f.wantExtra = w + } + } + if int(f.wantExtra)+128 > f.inodeSize { + f.wantExtra = uint16(f.inodeSize - 128) + } + // The superblock is in block 0 (4 KB blocks) or block 1 (1 KB). + f.sbBlock = uint64(1024 / f.bs) + f.sbOff = int(1024 % f.bs) + f.gdtBlock = f.firstData + 1 + return f, nil +} + +// Clean reports why the file system must not be changed now, or nil: it +// needs fsck, has a journal to replay, or orphans to finish. +func (f *FS) Clean() error { + sb, err := f.super() + if err != nil { + return err + } + state := le16(sb[0x3A:]) + switch { + case state&stateValid == 0 || state&stateErrors != 0: + return errors.New("ext4: the file system was not unmounted cleanly or has errors") + case le32(sb[0x60:])&incompatRecover != 0: + return errors.New("ext4: the journal has changes to replay") + case le32(sb[0xE8:]) != 0: + return errors.New("ext4: deleted files are still to be cleaned up") + } + return nil +} + +// block returns block n, read once and kept for changes. +func (f *FS) block(n uint64) ([]byte, error) { + if b, ok := f.cache[n]; ok { + return b, nil + } + if n >= f.blocks { + return nil, fmt.Errorf("%w: block %d beyond the end", ErrCorrupt, n) + } + b := make([]byte, f.bs) + if _, err := f.dev.ReadAt(b, f.offset+int64(n)*f.bs); err != nil { + return nil, err + } + f.cache[n] = b + return b, nil +} + +func (f *FS) markDirty(n uint64) { f.dirty[n] = true } + +func (f *FS) super() ([]byte, error) { + b, err := f.block(f.sbBlock) + if err != nil { + return nil, err + } + return b[f.sbOff : f.sbOff+1024], nil +} + +// desc returns group g's descriptor and the block it lives in. +func (f *FS) desc(g uint32) ([]byte, uint64, error) { + if g >= f.groups { + return nil, 0, fmt.Errorf("%w: group %d", ErrCorrupt, g) + } + off := uint64(g) * uint64(f.descSize) + n := f.gdtBlock + off/uint64(f.bs) + b, err := f.block(n) + if err != nil { + return nil, 0, err + } + o := off % uint64(f.bs) + return b[o : o+uint64(f.descSize)], n, nil +} + +func (f *FS) hi(d []byte, lo, hi int, wide bool) uint64 { + v := uint64(le32(d[lo:])) + if wide && f.descSize >= 64 { + v |= uint64(le32(d[hi:])) << 32 + } + return v +} + +func (f *FS) descBlockBitmap(d []byte) uint64 { return f.hi(d, 0x0, 0x20, true) } +func (f *FS) descInodeBitmap(d []byte) uint64 { return f.hi(d, 0x4, 0x24, true) } +func (f *FS) descInodeTable(d []byte) uint64 { return f.hi(d, 0x8, 0x28, true) } + +func (f *FS) count16(d []byte, lo, hi int) uint32 { + v := uint32(le16(d[lo:])) + if f.descSize >= 64 { + v |= uint32(le16(d[hi:])) << 16 + } + return v +} + +func (f *FS) setCount16(d []byte, lo, hi int, v uint32) { + put16(d[lo:], uint16(v)) + if f.descSize >= 64 { + put16(d[hi:], uint16(v>>16)) + } +} + +// inode is a copy of an inode's bytes with its number; save writes it +// back. +type inode struct { + ino uint32 + b []byte +} + +func (f *FS) inodeLoc(ino uint32) (uint64, int, error) { + if ino == 0 || ino > f.inodesPer*f.groups { + return 0, 0, fmt.Errorf("%w: inode %d", ErrCorrupt, ino) + } + g, i := (ino-1)/f.inodesPer, (ino-1)%f.inodesPer + d, _, err := f.desc(g) + if err != nil { + return 0, 0, err + } + off := uint64(i) * uint64(f.inodeSize) + return f.descInodeTable(d) + off/uint64(f.bs), int(off % uint64(f.bs)), nil +} + +func (f *FS) inode(ino uint32) (*inode, error) { + n, o, err := f.inodeLoc(ino) + if err != nil { + return nil, err + } + b, err := f.block(n) + if err != nil { + return nil, err + } + return &inode{ino: ino, b: append([]byte(nil), b[o:o+f.inodeSize]...)}, nil +} + +func (in *inode) mode() uint16 { return le16(in.b[0x0:]) } +func (in *inode) size() uint64 { return uint64(le32(in.b[0x4:])) | uint64(le32(in.b[0x6C:]))<<32 } +func (in *inode) flags() uint32 { return le32(in.b[0x20:]) } +func (in *inode) links() uint16 { return le16(in.b[0x1A:]) } +func (in *inode) gen() uint32 { return le32(in.b[0x64:]) } +func (in *inode) uid() uint32 { + return uint32(le16(in.b[0x2:])) | uint32(le16(in.b[0x78:]))<<16 +} +func (in *inode) gid() uint32 { + return uint32(le16(in.b[0x18:])) | uint32(le16(in.b[0x7A:]))<<16 +} + +// inodeSeed is the seed of the checksums of an inode and what belongs to +// it (directory blocks, extent blocks). +func (f *FS) inodeSeed(in *inode) uint32 { + var b [4]byte + put32(b[:], in.ino) + s := crc(f.csumSeed, b[:]) + put32(b[:], in.gen()) + return crc(s, b[:]) +} + +// extent is a run of blocks of a file: logical start, physical start, +// length; unwritten runs read as zeros. +type extent struct { + logical uint32 + physical uint64 + length uint32 + unwritten bool +} + +// extents lists a file's runs and the blocks its extent tree uses. +func (f *FS) extents(in *inode) (runs []extent, tree []uint64, err error) { + if in.flags()&inodeExtentsFl == 0 { + return nil, nil, fmt.Errorf("%w: inode %d does not use extents", ErrUnsupported, in.ino) + } + var walk func(node []byte, depth int) error + walk = func(node []byte, depth int) error { + if le16(node[0:]) != extentMagic || depth > 5 { + return fmt.Errorf("%w: extent tree of inode %d", ErrCorrupt, in.ino) + } + n, d := int(le16(node[2:])), int(le16(node[6:])) + if 12+12*n > len(node) { + return fmt.Errorf("%w: extent tree of inode %d", ErrCorrupt, in.ino) + } + for i := 0; i < n; i++ { + e := node[12+12*i:] + if d == 0 { + l := uint32(le16(e[4:])) + x := extent{logical: le32(e[0:]), physical: uint64(le16(e[6:]))<<32 | uint64(le32(e[8:])), length: l} + if l > 32768 { + x.length, x.unwritten = l-32768, true + } + runs = append(runs, x) + continue + } + child := uint64(le16(e[8:]))<<32 | uint64(le32(e[4:])) + b, err := f.block(child) + if err != nil { + return err + } + tree = append(tree, child) + if err := walk(b, depth+1); err != nil { + return err + } + } + return nil + } + return runs, tree, walk(in.b[0x28:0x28+60], 0) +} + +// read returns a file's content. +func (f *FS) read(in *inode) ([]byte, error) { + size := in.size() + if size > 1<<30 { + return nil, fmt.Errorf("ext4: inode %d is too large to read", in.ino) + } + out := make([]byte, size) + if in.mode()&modeType == modeSymlink && size < 60 && in.flags()&inodeExtentsFl == 0 { + copy(out, in.b[0x28:]) + return out, nil + } + runs, _, err := f.extents(in) + if err != nil { + return nil, err + } + for _, r := range runs { + if r.unwritten { + continue + } + for i := uint32(0); i < r.length; i++ { + at := (uint64(r.logical) + uint64(i)) * uint64(f.bs) + if at >= size { + break + } + b, err := f.block(r.physical + uint64(i)) + if err != nil { + return nil, err + } + copy(out[at:], b) + } + } + return out, nil +} + +// DirEntry is one name in a directory. +type DirEntry struct { + Name string + Inode uint32 + Type byte // 1 file, 2 directory, 7 symlink +} + +func (e DirEntry) IsDir() bool { return e.Type == 2 } + +// dirBlocks returns a directory's blocks in order. +func (f *FS) dirBlocks(in *inode) ([]uint64, error) { + runs, _, err := f.extents(in) + if err != nil { + return nil, err + } + n := int(in.size() / uint64(f.bs)) + out := make([]uint64, n) + for _, r := range runs { + for i := uint32(0); i < r.length; i++ { + if l := int(r.logical + i); l < n { + out[l] = r.physical + uint64(i) + } + } + } + for _, b := range out { + if b == 0 { + return nil, fmt.Errorf("%w: hole in directory %d", ErrCorrupt, in.ino) + } + } + return out, nil +} + +// entries walks the entries of one directory block; fn gets each entry's +// offset, record length and the entry itself (inode 0 for unused space). +func (f *FS) entries(b []byte, fn func(off, recLen int, e DirEntry) bool) error { + for off := 0; off < len(b); { + if off+8 > len(b) { + return fmt.Errorf("%w: directory entry", ErrCorrupt) + } + ino, recLen, nameLen := le32(b[off:]), int(le16(b[off+4:])), int(b[off+6]) + if recLen < 8 || recLen%4 != 0 || off+recLen > len(b) || 8+nameLen > recLen { + return fmt.Errorf("%w: directory entry", ErrCorrupt) + } + e := DirEntry{Inode: ino, Type: b[off+7], Name: string(b[off+8 : off+8+nameLen])} + if !fn(off, recLen, e) { + return nil + } + off += recLen + } + return nil +} + +func isTail(e DirEntry, recLen int) bool { + return e.Inode == 0 && recLen == 12 && e.Type == 0xDE && e.Name == "" +} + +func (f *FS) readDir(in *inode) ([]DirEntry, error) { + if in.mode()&modeType != modeDir { + return nil, fmt.Errorf("ext4: inode %d is not a directory", in.ino) + } + blocks, err := f.dirBlocks(in) + if err != nil { + return nil, err + } + var out []DirEntry + for _, n := range blocks { + b, err := f.block(n) + if err != nil { + return nil, err + } + err = f.entries(b, func(_, recLen int, e DirEntry) bool { + if e.Inode != 0 && e.Name != "." && e.Name != ".." && !isTail(e, recLen) { + out = append(out, e) + } + return true + }) + if err != nil { + return nil, err + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +// lookup follows an absolute path; symlinks are not followed. +func (f *FS) lookup(p string) (*inode, error) { + p = path.Clean("/" + p) + in, err := f.inode(rootIno) + if err != nil { + return nil, err + } + if p == "/" { + return in, nil + } + for _, part := range strings.Split(p[1:], "/") { + list, err := f.readDir(in) + if err != nil { + return nil, err + } + found := uint32(0) + for _, e := range list { + if e.Name == part { + found = e.Inode + break + } + } + if found == 0 { + return nil, fmt.Errorf("%s: %w", p, ErrNotExist) + } + if in, err = f.inode(found); err != nil { + return nil, err + } + } + return in, nil +} + +// ReadFile returns the content of a file (or a symlink's target). +func (f *FS) ReadFile(p string) ([]byte, error) { + in, err := f.lookup(p) + if err != nil { + return nil, err + } + if t := in.mode() & modeType; t != modeRegular && t != modeSymlink { + return nil, fmt.Errorf("ext4: %s is not a file", p) + } + return f.read(in) +} + +// ReadDir lists a directory, sorted by name. +func (f *FS) ReadDir(p string) ([]DirEntry, error) { + in, err := f.lookup(p) + if err != nil { + return nil, err + } + return f.readDir(in) +} + +// Info is what Stat tells about a path. +type Info struct { + Mode fs.FileMode + UID, GID uint32 + Size int64 + Links int +} + +// Stat describes a path without following a symlink at its end. +func (f *FS) Stat(p string) (Info, error) { + in, err := f.lookup(p) + if err != nil { + return Info{}, err + } + m := fs.FileMode(in.mode() & 0o7777) + switch in.mode() & modeType { + case modeDir: + m |= fs.ModeDir + case modeSymlink: + m |= fs.ModeSymlink + } + return Info{Mode: m, UID: in.uid(), GID: in.gid(), Size: int64(in.size()), Links: int(in.links())}, nil +} diff --git a/internal/ext4/ext4_test.go b/internal/ext4/ext4_test.go new file mode 100644 index 0000000..99349d7 --- /dev/null +++ b/internal/ext4/ext4_test.go @@ -0,0 +1,216 @@ +package ext4 + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// The tests make a file system the way Armbian's is made (mkfs.ext4 with +// metadata_csum, a checksum seed, 64bit, 4 KB blocks), change it, and let +// e2fsck judge the result; as root they also mount it and compare what +// Linux reads. They need e2fsprogs and run in `make test-linux`. + +func need(t *testing.T, tools ...string) { + for _, tool := range tools { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("no %s", tool) + } + } +} + +func sh(t *testing.T, name string, args ...string) string { + t.Helper() + out, err := exec.Command(name, args...).CombinedOutput() + if err != nil { + t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out) + } + return string(out) +} + +// mkfs makes an image of size MB from the files in src. +func mkfs(t *testing.T, src string, mb int) string { + img := filepath.Join(t.TempDir(), "root.img") + sh(t, "mkfs.ext4", "-q", "-F", "-b", "4096", "-L", "test", + "-O", "metadata_csum,metadata_csum_seed,64bit,^orphan_file", "-d", src, img, fmt.Sprintf("%dM", mb)) + return img +} + +// fsck runs e2fsck without changing anything; any finding fails the test. +func fsck(t *testing.T, img string) { + t.Helper() + out, err := exec.Command("e2fsck", "-fn", img).CombinedOutput() + if err != nil { + t.Fatalf("e2fsck found problems: %v\n%s", err, out) + } +} + +func openImage(t *testing.T, img string) (*FS, *os.File) { + t.Helper() + dev, err := os.OpenFile(img, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + f, err := Open(dev, 0) + if err != nil { + t.Fatal(err) + } + if err := f.Clean(); err != nil { + t.Fatal(err) + } + return f, dev +} + +func TestChanges(t *testing.T) { + need(t, "mkfs.ext4", "e2fsck") + src := t.TempDir() + os.MkdirAll(filepath.Join(src, "etc/systemd/system/multi-user.target.wants"), 0o755) + os.MkdirAll(filepath.Join(src, "boot"), 0o755) + os.MkdirAll(filepath.Join(src, "root"), 0o700) + os.WriteFile(filepath.Join(src, "etc/shadow"), []byte("root:$y$old:19000:0:99999:7:::\n"), 0o640) + os.WriteFile(filepath.Join(src, "root/.not_logged_in_yet"), []byte("1\n"), 0o644) + os.Symlink("/lib/systemd/system/armbian-resize-filesystem.service", + filepath.Join(src, "etc/systemd/system/multi-user.target.wants/armbian-resize-filesystem.service")) + // A directory too large for one block, which e2fsck -D indexes. + big := filepath.Join(src, "usr/share/many") + os.MkdirAll(big, 0o755) + for i := 0; i < 400; i++ { + os.WriteFile(filepath.Join(big, fmt.Sprintf("file-with-a-longish-name-%04d", i)), []byte{byte(i)}, 0o644) + } + img := mkfs(t, src, 96) + sh(t, "e2fsck", "-fyD", img) // index the large directory + fsck(t, img) + + f, dev := openImage(t, img) + if in, err := f.lookup("/usr/share/many"); err != nil || in.flags()&inodeIndexFl == 0 { + t.Fatalf("the large directory is not indexed: %v", err) + } + large := bytes.Repeat([]byte("0123456789abcdef"), 10<<20/16+3) // 10 MB and a bit + want := map[string][]byte{ + "/boot/squirrel/squirreld": large, + "/boot/squirrel/debs/xfsprogs_6.13.0_arm64.deb": bytes.Repeat([]byte{7}, 1<<20), + "/boot/squirrel/setup.env": []byte("TUSB_HOSTNAME='squirrel'\n"), + "/boot/squirrel/empty": {}, + "/etc/systemd/system/squirrel-firstboot.service": []byte("[Unit]\nDescription=first start\n"), + "/etc/shadow": []byte("root:!$y$old:19000:0:99999:7:::\nother:x:1:2::::\n"), + "/root/.no_rootfs_resize": {}, + "/usr/share/many/a-new-one": []byte("in an indexed directory"), + } + for _, d := range []string{"/boot/squirrel", "/boot/squirrel/debs"} { + if err := f.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + for p, data := range want { + mode, gid := os.FileMode(0o644), uint32(0) + if p == "/etc/shadow" { + mode, gid = 0o640, 42 + } + if err := f.WriteFile(p, data, mode, 0, gid); err != nil { + t.Fatalf("%s: %v", p, err) + } + } + for _, p := range []string{"/root/.not_logged_in_yet", "/etc/systemd/system/multi-user.target.wants/armbian-resize-filesystem.service"} { + if err := f.Remove(p); err != nil { + t.Fatalf("removing %s: %v", p, err) + } + } + if err := f.Remove("/root/not-there"); err == nil { + t.Error("removing a missing file succeeded") + } + wants := "/etc/systemd/system/multi-user.target.wants/squirrel-firstboot.service" + if err := f.Symlink("/etc/systemd/system/squirrel-firstboot.service", wants); err != nil { + t.Fatal(err) + } + long := "/" + strings.Repeat("a-long-target/", 6) + if err := f.Symlink(long, "/boot/squirrel/long-link"); err != nil { + t.Fatal(err) + } + // Replacing a file again, with something smaller. + want["/boot/squirrel/setup.env"] = []byte("x") + if err := f.WriteFile("/boot/squirrel/setup.env", want["/boot/squirrel/setup.env"], 0o600, 0, 0); err != nil { + t.Fatal(err) + } + if err := f.Flush(); err != nil { + t.Fatal(err) + } + dev.Close() + fsck(t, img) + + // Read back by a fresh open. + g, dev2 := openImage(t, img) + defer dev2.Close() + for p, data := range want { + got, err := g.ReadFile(p) + if err != nil || !bytes.Equal(got, data) { + t.Errorf("%s: %d bytes, want %d: %v", p, len(got), len(data), err) + } + } + if st, err := g.Stat("/etc/shadow"); err != nil || st.GID != 42 || st.Mode.Perm() != 0o640 { + t.Errorf("shadow: %+v %v", st, err) + } + for _, p := range []string{"/root/.not_logged_in_yet", "/etc/systemd/system/multi-user.target.wants/armbian-resize-filesystem.service"} { + if _, err := g.Stat(p); err == nil { + t.Errorf("%s is still there", p) + } + } + if got, _ := g.ReadFile(wants); string(got) != "/etc/systemd/system/squirrel-firstboot.service" { + t.Errorf("symlink reads %q", got) + } + if many, _ := g.ReadDir("/usr/share/many"); len(many) != 401 { + t.Errorf("indexed directory has %d entries", len(many)) + } + + // What Linux itself reads. + if os.Getuid() != 0 { + return + } + mnt := t.TempDir() + sh(t, "mount", "-o", "loop,ro", img, mnt) + defer exec.Command("umount", mnt).Run() + for p, data := range want { + got, err := os.ReadFile(filepath.Join(mnt, p)) + if err != nil || !bytes.Equal(got, data) { + t.Errorf("Linux reads %s: %d bytes, want %d: %v", p, len(got), len(data), err) + } + } + if target, err := os.Readlink(filepath.Join(mnt, "boot/squirrel/long-link")); err != nil || target != long { + t.Errorf("Linux reads the long link as %q: %v", target, err) + } + if many, err := os.ReadDir(filepath.Join(mnt, "usr/share/many")); err != nil || len(many) != 401 { + t.Errorf("Linux lists %d entries: %v", len(many), err) + } +} + +// A file system with a journal to replay, or one that is not clean, is +// left alone. +func TestRefusesUnclean(t *testing.T) { + need(t, "mkfs.ext4", "debugfs") + img := mkfs(t, t.TempDir(), 16) + sh(t, "debugfs", "-w", "-R", "ssv state 0", img) + dev, _ := os.Open(img) + defer dev.Close() + f, err := Open(dev, 0) + if err != nil { + t.Fatal(err) + } + if f.Clean() == nil { + t.Error("a file system that is not clean counts as clean") + } +} + +// A feature this package does not know is refused rather than ignored. +func TestRefusesUnknownFeatures(t *testing.T) { + need(t, "mkfs.ext4") + img := filepath.Join(t.TempDir(), "x.img") + sh(t, "mkfs.ext4", "-q", "-F", "-b", "4096", "-O", "inline_data,^orphan_file", img, "16M") + dev, _ := os.Open(img) + defer dev.Close() + if _, err := Open(dev, 0); err == nil { + t.Error("inline_data accepted") + } +} diff --git a/internal/ext4/write.go b/internal/ext4/write.go new file mode 100644 index 0000000..5344588 --- /dev/null +++ b/internal/ext4/write.go @@ -0,0 +1,880 @@ +package ext4 + +import ( + "crypto/rand" + "encoding/binary" + "fmt" + "io/fs" + "path" +) + +// Changes: files are written whole, into blocks allocated in one go; +// every changed block stays in memory and Flush writes them, after the +// checksums of the groups and the superblock are brought up to date. + +func (f *FS) tailSize() int { + if f.csum { + return 12 + } + return 0 +} + +func (f *FS) touch(g uint32) { + if f.touched == nil { + f.touched = map[uint32]bool{} + } + f.touched[g] = true +} + +// sbAdd adds d to the superblock's free block (or inode) count. +func (f *FS) sbAdd(blocks bool, d int64) error { + sb, err := f.super() + if err != nil { + return err + } + if blocks { + v := uint64(le32(sb[0xC:])) | uint64(le32(sb[0x158:]))<<32 + v = uint64(int64(v) + d) + put32(sb[0xC:], uint32(v)) + if f.descSize >= 64 { + put32(sb[0x158:], uint32(v>>32)) + } + } else { + put32(sb[0x10:], uint32(int64(le32(sb[0x10:]))+d)) + } + f.markDirty(f.sbBlock) + return nil +} + +func (f *FS) groupStart(g uint32) uint64 { return f.firstData + uint64(g)*uint64(f.perGroup) } + +func (f *FS) groupLen(g uint32) uint32 { + start := f.groupStart(g) + if start+uint64(f.perGroup) > f.blocks { + return uint32(f.blocks - start) + } + return f.perGroup +} + +func bit(b []byte, i uint32) bool { return b[i/8]&(1<<(i%8)) != 0 } +func setBit(b []byte, i uint32, on bool) { + if on { + b[i/8] |= 1 << (i % 8) + } else { + b[i/8] &^= 1 << (i % 8) + } +} + +// allocBlocks finds n free blocks: one run if it can, else up to max +// runs, in groups whose bitmap is initialized. +func (f *FS) allocBlocks(n uint32, max int) ([]extent, error) { + if n == 0 { + return nil, nil + } + var free []run + for g := uint32(0); g < f.groups; g++ { + d, _, err := f.desc(g) + if err != nil { + return nil, err + } + if le16(d[0x12:])&bgBlockUninit != 0 || f.count16(d, 0xC, 0x2C) == 0 { + continue + } + bm, err := f.block(f.descBlockBitmap(d)) + if err != nil { + return nil, err + } + l := f.groupLen(g) + for i := uint32(0); i < l; { + if bit(bm, i) { + i++ + continue + } + j := i + for j < l && !bit(bm, j) { + j++ + } + free = append(free, run{g, i, j - i}) + if j-i >= n { + return f.take([]run{{g, i, n}}) + } + i = j + } + } + // No single run: the largest runs first. + var picked []run + need := n + for len(picked) < max && need > 0 { + best := -1 + for k, r := range free { + if r.len > 0 && (best < 0 || r.len > free[best].len) { + best = k + } + } + if best < 0 { + break + } + r := free[best] + r.len = min(r.len, need) + picked = append(picked, r) + need -= r.len + free[best].len = 0 + } + if need > 0 { + return nil, ErrNoSpace + } + return f.take(picked) +} + +// run is a stretch of free blocks in a group's bitmap. +type run struct { + g uint32 + start uint32 + len uint32 +} + +// take marks runs as used. +func (f *FS) take(runs []run) ([]extent, error) { + var out []extent + for _, r := range runs { + d, n, err := f.desc(r.g) + if err != nil { + return nil, err + } + bmBlock := f.descBlockBitmap(d) + bm, err := f.block(bmBlock) + if err != nil { + return nil, err + } + for i := r.start; i < r.start+r.len; i++ { + setBit(bm, i, true) + } + f.markDirty(bmBlock) + f.setCount16(d, 0xC, 0x2C, f.count16(d, 0xC, 0x2C)-r.len) + f.markDirty(n) + f.touch(r.g) + if err := f.sbAdd(true, -int64(r.len)); err != nil { + return nil, err + } + out = append(out, extent{physical: f.groupStart(r.g) + uint64(r.start), length: r.len}) + } + return out, nil +} + +func (f *FS) freeBlocks(start uint64, length uint32) error { + for b := start; b < start+uint64(length); b++ { + g := uint32((b - f.firstData) / uint64(f.perGroup)) + i := uint32((b - f.firstData) % uint64(f.perGroup)) + d, n, err := f.desc(g) + if err != nil { + return err + } + bmBlock := f.descBlockBitmap(d) + bm, err := f.block(bmBlock) + if err != nil { + return err + } + if !bit(bm, i) { + return fmt.Errorf("%w: block %d freed twice", ErrCorrupt, b) + } + setBit(bm, i, false) + f.markDirty(bmBlock) + f.setCount16(d, 0xC, 0x2C, f.count16(d, 0xC, 0x2C)+1) + f.markDirty(n) + f.touch(g) + if err := f.sbAdd(true, 1); err != nil { + return err + } + } + return nil +} + +// allocInode takes a free inode, near group pref if it can. +func (f *FS) allocInode(pref uint32, dir bool) (uint32, error) { + for k := uint32(0); k < f.groups; k++ { + g := (pref + k) % f.groups + d, n, err := f.desc(g) + if err != nil { + return 0, err + } + if le16(d[0x12:])&bgInodeUninit != 0 || f.count16(d, 0xE, 0x2E) == 0 { + continue + } + bmBlock := f.descInodeBitmap(d) + bm, err := f.block(bmBlock) + if err != nil { + return 0, err + } + for i := uint32(0); i < f.inodesPer; i++ { + ino := g*f.inodesPer + i + 1 + if ino < f.firstIno || bit(bm, i) { + continue + } + setBit(bm, i, true) + f.markDirty(bmBlock) + f.setCount16(d, 0xE, 0x2E, f.count16(d, 0xE, 0x2E)-1) + if dir { + f.setCount16(d, 0x10, 0x30, f.count16(d, 0x10, 0x30)+1) + } + // Inodes past the last one in use are not checked; this one is + // in use now. + if unused := f.count16(d, 0x1C, 0x32); i >= f.inodesPer-unused { + f.setCount16(d, 0x1C, 0x32, f.inodesPer-i-1) + } + f.markDirty(n) + f.touch(g) + return ino, f.sbAdd(false, -1) + } + } + return 0, ErrNoSpace +} + +func (f *FS) freeInode(in *inode) error { + g, i := (in.ino-1)/f.inodesPer, (in.ino-1)%f.inodesPer + d, n, err := f.desc(g) + if err != nil { + return err + } + bmBlock := f.descInodeBitmap(d) + bm, err := f.block(bmBlock) + if err != nil { + return err + } + setBit(bm, i, false) + f.markDirty(bmBlock) + f.setCount16(d, 0xE, 0x2E, f.count16(d, 0xE, 0x2E)+1) + if in.mode()&modeType == modeDir { + f.setCount16(d, 0x10, 0x30, f.count16(d, 0x10, 0x30)-1) + } + f.markDirty(n) + f.touch(g) + put16(in.b[0x1A:], 0) + put32(in.b[0x14:], f.now) // dtime + if err := f.save(in); err != nil { + return err + } + return f.sbAdd(false, 1) +} + +// save writes an inode back into the inode table, with its checksum. +func (f *FS) save(in *inode) error { + if f.csum { + c := append([]byte(nil), in.b...) + put16(c[0x7C:], 0) + extra := 0 + if f.inodeSize > 128 { + extra = int(le16(c[0x80:])) + } + if extra >= 4 { + put16(c[0x82:], 0) + } + v := crc(f.inodeSeed(in), c) + put16(in.b[0x7C:], uint16(v)) + if extra >= 4 { + put16(in.b[0x82:], uint16(v>>16)) + } + } + n, o, err := f.inodeLoc(in.ino) + if err != nil { + return err + } + b, err := f.block(n) + if err != nil { + return err + } + copy(b[o:o+f.inodeSize], in.b) + f.markDirty(n) + return nil +} + +func (f *FS) newInode(ino uint32, mode uint16, uid, gid uint32) *inode { + in := &inode{ino: ino, b: make([]byte, f.inodeSize)} + put16(in.b[0x0:], mode) + put16(in.b[0x2:], uint16(uid)) + put16(in.b[0x78:], uint16(uid>>16)) + put16(in.b[0x18:], uint16(gid)) + put16(in.b[0x7A:], uint16(gid>>16)) + for _, o := range []int{0x8, 0xC, 0x10} { + put32(in.b[o:], f.now) + } + put16(in.b[0x1A:], 1) + var g [4]byte + rand.Read(g[:]) + put32(in.b[0x64:], binary.LittleEndian.Uint32(g[:])) + if f.inodeSize > 128 { + put16(in.b[0x80:], f.wantExtra) + if f.wantExtra >= 0x18 { + put32(in.b[0x90:], f.now) // crtime + } + } + return in +} + +func (in *inode) setSize(n uint64) { + put32(in.b[0x4:], uint32(n)) + put32(in.b[0x6C:], uint32(n>>32)) +} + +func (f *FS) setBlockCount(in *inode, blocks uint64) { + sectors := blocks * uint64(f.bs) / 512 + put32(in.b[0x1C:], uint32(sectors)) + put16(in.b[0x74:], uint16(sectors>>32)) + put32(in.b[0x20:], in.flags()&^inodeHugeFl) +} + +// freeData gives back a file's blocks, the extent tree's included. +func (f *FS) freeData(in *inode) error { + if in.flags()&inodeExtentsFl == 0 { + return nil // a fast symlink: the target is in the inode + } + runs, tree, err := f.extents(in) + if err != nil { + return err + } + for _, r := range runs { + if err := f.freeBlocks(r.physical, r.length); err != nil { + return err + } + } + for _, b := range tree { + if err := f.freeBlocks(b, 1); err != nil { + return err + } + } + return nil +} + +// setExtents puts runs (logical order) into an inode: up to four in the +// inode itself, more in one leaf block. +func (f *FS) setExtents(in *inode, runs []extent) (treeBlocks uint64, err error) { + ib := in.b[0x28 : 0x28+60] + clear(ib) + put32(in.b[0x20:], in.flags()|inodeExtentsFl) + put16(ib[0:], extentMagic) + put16(ib[4:], 4) + putExtent := func(e []byte, r extent) { + put32(e[0:], r.logical) + put16(e[4:], uint16(r.length)) + put16(e[6:], uint16(r.physical>>32)) + put32(e[8:], uint32(r.physical)) + } + for _, r := range runs { + if r.length > 32768 { + return 0, fmt.Errorf("ext4: run of %d blocks", r.length) + } + } + if len(runs) <= 4 { + put16(ib[2:], uint16(len(runs))) + for i, r := range runs { + putExtent(ib[12+12*i:], r) + } + return 0, nil + } + max := (int(f.bs) - 12) / 12 + if len(runs) > max { + return 0, fmt.Errorf("ext4: %d runs is too fragmented", len(runs)) + } + leaf, err := f.allocBlocks(1, 1) + if err != nil { + return 0, err + } + lb, err := f.block(leaf[0].physical) + if err != nil { + return 0, err + } + clear(lb) + put16(lb[0:], extentMagic) + put16(lb[2:], uint16(len(runs))) + put16(lb[4:], uint16(max)) + for i, r := range runs { + putExtent(lb[12+12*i:], r) + } + if f.csum { + put32(lb[12+12*max:], crc(f.inodeSeed(in), lb[:12+12*max])) + } + f.markDirty(leaf[0].physical) + put16(ib[2:], 1) + put16(ib[6:], 1) // depth + put32(ib[12:], 0) + put32(ib[16:], uint32(leaf[0].physical)) + put16(ib[20:], uint16(leaf[0].physical>>32)) + return 1, nil +} + +// fill allocates blocks for data, writes it and sets the inode's extents, +// size and block count. +func (f *FS) fill(in *inode, data []byte) error { + n := uint32((int64(len(data)) + f.bs - 1) / f.bs) + runs, err := f.allocBlocks(n, 64) + if err != nil { + return err + } + var logical uint32 + for i := range runs { + runs[i].logical = logical + for k := uint32(0); k < runs[i].length; k++ { + b, err := f.block(runs[i].physical + uint64(k)) + if err != nil { + return err + } + clear(b) + copy(b, data[min(int64(logical+k)*f.bs, int64(len(data))):]) + f.markDirty(runs[i].physical + uint64(k)) + } + logical += runs[i].length + } + tree, err := f.setExtents(in, runs) + if err != nil { + return err + } + in.setSize(uint64(len(data))) + f.setBlockCount(in, uint64(n)+tree) + return nil +} + +func (f *FS) group(ino uint32) uint32 { return (ino - 1) / f.inodesPer } + +func fileType(mode uint16) byte { + switch mode & modeType { + case modeDir: + return 2 + case modeSymlink: + return 7 + } + return 1 +} + +// dirBlockCsum sets the checksum tail of a directory block. +func (f *FS) dirBlockCsum(dir *inode, b []byte) { + if !f.csum { + return + } + t := b[len(b)-12:] + put32(t[0:], 0) + put16(t[4:], 12) + t[6], t[7] = 0, 0xDE + put32(t[8:], crc(f.inodeSeed(dir), b[:len(b)-12])) +} + +func recLenFor(name string) int { return (8 + len(name) + 3) &^ 3 } + +func putEntry(b []byte, ino uint32, recLen int, name string, typ byte) { + put32(b[0:], ino) + put16(b[4:], uint16(recLen)) + b[6], b[7] = byte(len(name)), typ + copy(b[8:], name) +} + +// unindex turns an indexed (htree) directory into a plain one, so an +// entry can go into any block. Its leaf blocks are plain blocks already; +// the root and the index nodes become empty ones. +func (f *FS) unindex(dir *inode) error { + blocks, err := f.dirBlocks(dir) + if err != nil { + return err + } + root, err := f.block(blocks[0]) + if err != nil { + return err + } + levels := int(root[30]) + if levels > 2 { + return fmt.Errorf("%w: directory index of %d levels", ErrUnsupported, levels) + } + index := map[uint32]bool{} + children := func(b []byte, at int) []uint32 { + count := int(le16(b[at+2:])) + out := []uint32{le32(b[at+4:])} + for i := 1; i < count && at+8*i+8 <= len(b); i++ { + out = append(out, le32(b[at+8*i+4:])) + } + return out + } + level := children(root, 0x20) + for l := 0; l < levels; l++ { + var next []uint32 + for _, c := range level { + if int(c) >= len(blocks) { + return fmt.Errorf("%w: directory index", ErrCorrupt) + } + index[c] = true + b, err := f.block(blocks[c]) + if err != nil { + return err + } + next = append(next, children(b, 8)...) + } + level = next + } + parent := le32(root[12:]) + clear(root) + putEntry(root, dir.ino, 12, ".", 2) + putEntry(root[12:], parent, len(root)-12-f.tailSize(), "..", 2) + f.dirBlockCsum(dir, root) + f.markDirty(blocks[0]) + for c := range index { + b, err := f.block(blocks[c]) + if err != nil { + return err + } + clear(b) + put16(b[4:], uint16(len(b)-f.tailSize())) + f.dirBlockCsum(dir, b) + f.markDirty(blocks[c]) + } + put32(dir.b[0x20:], dir.flags()&^inodeIndexFl) + return f.save(dir) +} + +// dirAdd adds name -> ino to a directory, in a free gap or a new block. +func (f *FS) dirAdd(dir *inode, name string, ino uint32, typ byte) error { + if len(name) == 0 || len(name) > 255 { + return fmt.Errorf("ext4: name %q", name) + } + if dir.flags()&inodeIndexFl != 0 { + if err := f.unindex(dir); err != nil { + return err + } + } + blocks, err := f.dirBlocks(dir) + if err != nil { + return err + } + need := recLenFor(name) + for _, n := range blocks { + b, err := f.block(n) + if err != nil { + return err + } + done := false + err = f.entries(b, func(off, recLen int, e DirEntry) bool { + if isTail(e, recLen) { + return true + } + used := 0 + if e.Inode != 0 { + used = recLenFor(e.Name) + } + if recLen-used < need { + return true + } + if used > 0 { + put16(b[off+4:], uint16(used)) + } + putEntry(b[off+used:], ino, recLen-used, name, typ) + done = true + return false + }) + if err != nil { + return err + } + if done { + f.dirBlockCsum(dir, b) + f.markDirty(n) + return nil + } + } + // A new block at the end of the directory. + runs, tree, err := f.extents(dir) + if err != nil { + return err + } + nb, err := f.allocBlocks(1, 1) + if err != nil { + return err + } + for _, t := range tree { + if err := f.freeBlocks(t, 1); err != nil { + return err + } + } + nb[0].logical = uint32(len(blocks)) + if last := len(runs) - 1; last >= 0 && runs[last].physical+uint64(runs[last].length) == nb[0].physical && + runs[last].logical+runs[last].length == nb[0].logical && !runs[last].unwritten && runs[last].length < 32768 { + runs[last].length++ + } else { + runs = append(runs, nb[0]) + } + b, err := f.block(nb[0].physical) + if err != nil { + return err + } + clear(b) + putEntry(b, ino, len(b)-f.tailSize(), name, typ) + f.dirBlockCsum(dir, b) + f.markDirty(nb[0].physical) + t, err := f.setExtents(dir, runs) + if err != nil { + return err + } + dir.setSize(uint64(len(blocks)+1) * uint64(f.bs)) + f.setBlockCount(dir, uint64(len(blocks)+1)+t) + return f.save(dir) +} + +// dirRemove takes name out of a directory and returns its inode number. +func (f *FS) dirRemove(dir *inode, name string) (uint32, error) { + if dir.flags()&inodeIndexFl != 0 { + if err := f.unindex(dir); err != nil { + return 0, err + } + } + blocks, err := f.dirBlocks(dir) + if err != nil { + return 0, err + } + for _, n := range blocks { + b, err := f.block(n) + if err != nil { + return 0, err + } + var found uint32 + prev := -1 + err = f.entries(b, func(off, recLen int, e DirEntry) bool { + if e.Inode != 0 && e.Name == name && !isTail(e, recLen) { + found = e.Inode + if prev >= 0 { + put16(b[prev+4:], le16(b[prev+4:])+uint16(recLen)) + } else { + put32(b[off:], 0) + } + return false + } + prev = off + return true + }) + if err != nil { + return 0, err + } + if found != 0 { + f.dirBlockCsum(dir, b) + f.markDirty(n) + return found, nil + } + } + return 0, ErrNotExist +} + +func (f *FS) parent(p string) (*inode, string, error) { + p = path.Clean("/" + p) + if p == "/" { + return nil, "", fmt.Errorf("ext4: cannot change /") + } + dir, err := f.lookup(path.Dir(p)) + if err != nil { + return nil, "", err + } + if dir.mode()&modeType != modeDir { + return nil, "", fmt.Errorf("ext4: %s is not a directory", path.Dir(p)) + } + return dir, path.Base(p), nil +} + +// WriteFile writes a file, replacing one that is there (in place, keeping +// its inode, so its directory does not change). +func (f *FS) WriteFile(p string, data []byte, perm fs.FileMode, uid, gid uint32) error { + dir, name, err := f.parent(p) + if err != nil { + return err + } + mode := modeRegular | uint16(perm&0o7777) + if old, err := f.lookup(p); err == nil { + if old.mode()&modeType == modeRegular { + if err := f.freeData(old); err != nil { + return err + } + put16(old.b[0x0:], mode) + put16(old.b[0x2:], uint16(uid)) + put16(old.b[0x78:], uint16(uid>>16)) + put16(old.b[0x18:], uint16(gid)) + put16(old.b[0x7A:], uint16(gid>>16)) + put32(old.b[0xC:], f.now) + put32(old.b[0x10:], f.now) + if err := f.fill(old, data); err != nil { + return err + } + return f.save(old) + } + if err := f.Remove(p); err != nil { + return err + } + } + ino, err := f.allocInode(f.group(dir.ino), false) + if err != nil { + return err + } + in := f.newInode(ino, mode, uid, gid) + if err := f.fill(in, data); err != nil { + return err + } + if err := f.save(in); err != nil { + return err + } + return f.dirAdd(dir, name, ino, 1) +} + +// Mkdir makes a directory; one that is there already is fine. +func (f *FS) Mkdir(p string, perm fs.FileMode) error { + if old, err := f.lookup(p); err == nil { + if old.mode()&modeType == modeDir { + return nil + } + return fmt.Errorf("ext4: %s exists and is not a directory", p) + } + dir, name, err := f.parent(p) + if err != nil { + return err + } + ino, err := f.allocInode(f.group(dir.ino), true) + if err != nil { + return err + } + in := f.newInode(ino, modeDir|uint16(perm&0o7777), 0, 0) + put16(in.b[0x1A:], 2) + blk, err := f.allocBlocks(1, 1) + if err != nil { + return err + } + b, err := f.block(blk[0].physical) + if err != nil { + return err + } + clear(b) + putEntry(b, ino, 12, ".", 2) + putEntry(b[12:], dir.ino, len(b)-12-f.tailSize(), "..", 2) + f.dirBlockCsum(in, b) + f.markDirty(blk[0].physical) + if _, err := f.setExtents(in, blk); err != nil { + return err + } + in.setSize(uint64(f.bs)) + f.setBlockCount(in, 1) + if err := f.save(in); err != nil { + return err + } + if err := f.dirAdd(dir, name, ino, 2); err != nil { + return err + } + // The new directory's ".." links to the parent. + if l := dir.links(); l > 1 && l < 64999 { + put16(dir.b[0x1A:], l+1) + } + put32(dir.b[0xC:], f.now) + put32(dir.b[0x10:], f.now) + return f.save(dir) +} + +// Symlink makes p a symlink to target; p must not exist. +func (f *FS) Symlink(target, p string) error { + if _, err := f.lookup(p); err == nil { + return fmt.Errorf("ext4: %s exists", p) + } + dir, name, err := f.parent(p) + if err != nil { + return err + } + ino, err := f.allocInode(f.group(dir.ino), false) + if err != nil { + return err + } + in := f.newInode(ino, modeSymlink|0o777, 0, 0) + if len(target) < 60 { + copy(in.b[0x28:], target) // a fast symlink, in the inode + in.setSize(uint64(len(target))) + } else if err := f.fill(in, []byte(target)); err != nil { + return err + } + if err := f.save(in); err != nil { + return err + } + return f.dirAdd(dir, name, ino, 7) +} + +// Remove removes a file or symlink; ErrNotExist when it is not there. +func (f *FS) Remove(p string) error { + in, err := f.lookup(p) + if err != nil { + return err + } + if in.mode()&modeType == modeDir { + return fmt.Errorf("ext4: %s is a directory", p) + } + dir, name, err := f.parent(p) + if err != nil { + return err + } + if _, err := f.dirRemove(dir, name); err != nil { + return err + } + put32(dir.b[0xC:], f.now) + put32(dir.b[0x10:], f.now) + if err := f.save(dir); err != nil { + return err + } + if l := in.links(); l > 1 { + put16(in.b[0x1A:], l-1) + return f.save(in) + } + if err := f.freeData(in); err != nil { + return err + } + return f.freeInode(in) +} + +// Flush brings the checksums of the changed groups and the superblock up +// to date and writes every changed block. +func (f *FS) Flush() error { + for g := range f.touched { + d, n, err := f.desc(g) + if err != nil { + return err + } + if f.csum { + bb, err := f.block(f.descBlockBitmap(d)) + if err != nil { + return err + } + v := crc(f.csumSeed, bb[:f.perGroup/8]) + put16(d[0x18:], uint16(v)) + if f.descSize >= 0x3A { + put16(d[0x38:], uint16(v>>16)) + } + ib, err := f.block(f.descInodeBitmap(d)) + if err != nil { + return err + } + v = crc(f.csumSeed, ib[:f.inodesPer/8]) + put16(d[0x1A:], uint16(v)) + if f.descSize >= 0x3C { + put16(d[0x3A:], uint16(v>>16)) + } + var gb [4]byte + put32(gb[:], g) + c := crc(f.csumSeed, gb[:]) + c = crc(c, d[:0x1E]) + c = crc(c, []byte{0, 0}) + if f.descSize > 0x20 { + c = crc(c, d[0x20:f.descSize]) + } + put16(d[0x1E:], uint16(c)) + } + f.markDirty(n) + } + sb, err := f.super() + if err != nil { + return err + } + put32(sb[0x30:], f.now) // last write + if f.csum { + put32(sb[0x3FC:], crc(^uint32(0), sb[:0x3FC])) + } + f.markDirty(f.sbBlock) + for n := range f.dirty { + if _, err := f.dev.WriteAt(f.cache[n], f.offset+int64(n)*f.bs); err != nil { + return fmt.Errorf("ext4: writing block %d: %w", n, err) + } + } + f.dirty = map[uint64]bool{} + f.touched = nil + return nil +} diff --git a/internal/maker/armbian.go b/internal/maker/armbian.go index 66d1d1d..a60d2cb 100644 --- a/internal/maker/armbian.go +++ b/internal/maker/armbian.go @@ -8,17 +8,17 @@ import ( "fmt" "io/fs" "os" - "os/exec" "path" - "path/filepath" "sort" "strings" + + "github.com/marcusaram/squirrel/internal/ext4" ) // Armbian has no FAT partition: everything, /boot included, is on one -// ext4 root. The maker puts its files there with debugfs (from e2fsprogs, -// built for macOS and carried in the app), which edits the file system -// without mounting it, and checks the result with e2fsck. +// ext4 root. The maker puts its files there with the ext4 package, which +// changes the file system without mounting it, on macOS and Windows +// alike. // // What goes on the card: // - /boot/squirrel: squirreld, the scripts, the packages Armbian minimal @@ -193,7 +193,10 @@ var armbianRemove = []string{ "/etc/systemd/system/serial-getty@.service.d/override.conf", } -const firstbootWants = "/etc/systemd/system/multi-user.target.wants/squirrel-firstboot.service" +const ( + firstbootUnit = "/etc/systemd/system/squirrel-firstboot.service" + firstbootWants = "/etc/systemd/system/multi-user.target.wants/squirrel-firstboot.service" +) // lockRoot locks root's password (1234 on Armbian) in /etc/shadow. func lockRoot(shadow []byte) ([]byte, error) { @@ -211,64 +214,37 @@ func lockRoot(shadow []byte) ([]byte, error) { return nil, errors.New("no root entry in /etc/shadow") } -type debugfs struct { - bin, e2fsck, image string - // dev, when set, is an opened card (from authopen); the tools reach - // it as /dev/fd/3, as they may not open the disk themselves. - dev *os.File -} - -func (d debugfs) target() string { - if d.dev != nil { - return fmt.Sprintf("/dev/fd/3?offset=%d", armbianRootStart) - } - return fmt.Sprintf("%s?offset=%d", d.image, armbianRootStart) +// root is Armbian's ext4 root, on an image or a card, changed by the ext4 +// package without mounting it. +type root struct { + dev ext4.Device + fs *ext4.FS } -// tool runs debugfs or e2fsck on the image or card. -func (d debugfs) tool(ctx context.Context, bin string, args ...string) ([]byte, error) { - cmd := exec.CommandContext(ctx, bin, append(args, d.target())...) - if d.dev != nil { - cmd.ExtraFiles = []*os.File{d.dev} - } - out, err := cmd.CombinedOutput() +// openRoot opens the root and refuses one that is not clean or uses ext4 +// features the ext4 package does not handle. +func openRoot(dev ext4.Device) (*root, error) { + f, err := ext4.Open(dev, armbianRootStart) if err != nil { - return out, fmt.Errorf("%s: %v: %s", filepath.Base(bin), err, bytes.TrimSpace(out)) + return nil, err } - return out, nil -} - -func (d debugfs) run(ctx context.Context, cmd string) ([]byte, error) { - return d.tool(ctx, d.bin, "-R", cmd) + if err := f.Clean(); err != nil { + return nil, fmt.Errorf("the card's file system cannot be changed now: %w", err) + } + return &root{dev: dev, fs: f}, nil } -func (d debugfs) exists(ctx context.Context, p string) bool { - out, err := d.run(ctx, "stat "+p) - return err == nil && bytes.Contains(out, []byte("Inode:")) +func (r *root) exists(p string) bool { + _, err := r.fs.Stat(p) + return err == nil } -// read returns a file from the image, or an error when it does not exist. -func (d debugfs) read(ctx context.Context, p string) ([]byte, error) { - tmp, err := os.CreateTemp("", "squirrel-maker-*") - if err != nil { - return nil, err - } - tmp.Close() - defer os.Remove(tmp.Name()) - os.Remove(tmp.Name()) - if _, err := d.run(ctx, "dump "+p+" "+tmp.Name()); err != nil { - return nil, err - } - data, err := os.ReadFile(tmp.Name()) - if err != nil { - return nil, fmt.Errorf("%s is not in the image", p) - } - return data, nil -} +// read returns a file from the root, or an error when it does not exist. +func (r *root) read(p string) ([]byte, error) { return r.fs.ReadFile(p) } // CustomizeArmbian puts the settings and payload on the root of an -// uncompressed Armbian image. tools holds debugfs and e2fsck. -func CustomizeArmbian(ctx context.Context, image string, s Settings, payload fs.FS, tools string, report Progress) error { +// uncompressed Armbian image. +func CustomizeArmbian(ctx context.Context, image string, s Settings, payload fs.FS, report Progress) error { if err := CheckArmbianLayout(image); err != nil { return err } @@ -276,17 +252,21 @@ func CustomizeArmbian(ctx context.Context, image string, s Settings, payload fs. if err != nil { return err } - d := debugfs{bin: filepath.Join(tools, "debugfs"), e2fsck: filepath.Join(tools, "e2fsck"), image: image} report("Adapting the image", -1) - - if out, err := d.tool(ctx, d.e2fsck, "-fn"); err != nil { - return fmt.Errorf("the image's file system is not clean before changing it: %s", lastLine(out)) + img, err := os.OpenFile(image, os.O_RDWR, 0) + if err != nil { + return err } - release, err := d.read(ctx, "/etc/armbian-release") + defer img.Close() + r, err := openRoot(img) + if err != nil { + return err + } + release, err := r.read("/etc/armbian-release") if err != nil || !bytes.Contains(release, []byte("BOARD=rockpi-4cplus")) { return errors.New("this does not look like Armbian for the ROCK 4C+") } - shadow, err := d.read(ctx, "/etc/shadow") + shadow, err := r.read("/etc/shadow") if err != nil { return err } @@ -295,63 +275,56 @@ func CustomizeArmbian(ctx context.Context, image string, s Settings, payload fs. } files["/etc/shadow"] = ext4File{data: shadow, mode: 0o640, gid: 42} // group shadow - return d.put(ctx, files, armbianRemove, []string{armbianDir, armbianDir + "/debs"}, true, report) + if err := r.put(files, armbianRemove, []string{armbianDir, armbianDir + "/debs"}, true, report); err != nil { + return err + } + return img.Sync() } // put writes files onto the root, removes the paths in remove (missing -// ones are fine), and checks the result with e2fsck and by reading -// everything back. dirs are created when missing. -func (d debugfs) put(ctx context.Context, files map[string]ext4File, remove, dirs []string, enable bool, report Progress) error { - staging, err := os.MkdirTemp("", "squirrel-maker-*") - if err != nil { - return err - } - defer os.RemoveAll(staging) - var script strings.Builder - // debugfs's mkdir on an existing directory leaves a stray one behind. +// ones are fine), and checks the result by opening the root afresh and +// reading everything back. dirs are created when missing. +func (r *root) put(files map[string]ext4File, remove, dirs []string, enable bool, report Progress) error { for _, dir := range dirs { - if !d.exists(ctx, dir) { - script.WriteString("mkdir " + dir + "\n") + if err := r.fs.Mkdir(dir, 0o755); err != nil { + return err } } for _, p := range remove { - script.WriteString("rm " + p + "\n") + if err := r.fs.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("removing %s: %w", p, err) + } } names := make([]string, 0, len(files)) for p := range files { names = append(names, p) } sort.Strings(names) - for i, p := range names { - if strings.ContainsAny(p, " \t\n\"") { - return fmt.Errorf("unexpected path %q", p) - } - src := filepath.Join(staging, fmt.Sprint(i)) - if err := os.WriteFile(src, files[p].data, 0o600); err != nil { - return err + for _, p := range names { + if err := r.fs.WriteFile(p, files[p].data, files[p].mode.Perm(), 0, uint32(files[p].gid)); err != nil { + return fmt.Errorf("writing %s: %w", p, err) } - fmt.Fprintf(&script, "rm %s\nwrite %s %s\nsif %s mode 0100%o\nsif %s uid 0\nsif %s gid %d\n", - p, src, p, p, files[p].mode.Perm(), p, p, files[p].gid) } if enable { - fmt.Fprintf(&script, "rm %s\nsymlink %s /etc/systemd/system/squirrel-firstboot.service\n", firstbootWants, firstbootWants) - } - scriptPath := filepath.Join(staging, "script") - if err := os.WriteFile(scriptPath, []byte(script.String()), 0o600); err != nil { - return err + if err := r.fs.Remove(firstbootWants); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + if err := r.fs.Symlink(firstbootUnit, firstbootWants); err != nil { + return err + } } - // debugfs carries on after a failing command (rm of a file that is not - // there is fine); what matters is checked below. - if _, err := d.tool(ctx, d.bin, "-w", "-f", scriptPath); err != nil { + if err := r.fs.Flush(); err != nil { return err } + // Read back from the device, not from what is still in memory. report("Checking the image", -1) - if out, err := d.tool(ctx, d.e2fsck, "-fn"); err != nil { - return fmt.Errorf("the image's file system has errors after changing it: %s", lastLine(out)) + check, err := openRoot(r.dev) + if err != nil { + return fmt.Errorf("checking: %w", err) } for _, p := range names { - got, err := d.read(ctx, p) + got, err := check.read(p) if err != nil { return fmt.Errorf("checking %s: %w", p, err) } @@ -360,20 +333,15 @@ func (d debugfs) put(ctx context.Context, files map[string]ext4File, remove, dir } } for _, p := range remove { - if _, err := d.read(ctx, p); err == nil { + if check.exists(p) { return fmt.Errorf("checking: %s is still there", p) } } if !enable { return nil } - if out, err := d.run(ctx, "stat "+firstbootWants); err != nil || !bytes.Contains(out, []byte("Type: symlink")) { + if target, err := check.read(firstbootWants); err != nil || string(target) != firstbootUnit { return errors.New("checking: squirrel-firstboot.service is not enabled") } return nil } - -func lastLine(out []byte) string { - lines := strings.Split(strings.TrimSpace(string(out)), "\n") - return lines[len(lines)-1] -} diff --git a/internal/maker/armbian_test.go b/internal/maker/armbian_test.go index 34d2ff7..1ceda38 100644 --- a/internal/maker/armbian_test.go +++ b/internal/maker/armbian_test.go @@ -2,6 +2,7 @@ package maker import ( "context" + "fmt" "os" "os/exec" "path/filepath" @@ -51,16 +52,28 @@ func TestArmbianSetupInBash(t *testing.T) { } } +// fsckRoot lets e2fsck, where it is installed, judge the image's root +// after the ext4 package changed it. +func fsckRoot(t *testing.T, img string) { + t.Helper() + if _, err := exec.LookPath("e2fsck"); err != nil { + return + } + target := fmt.Sprintf("%s?offset=%d", img, armbianRootStart) + if out, err := exec.Command("e2fsck", "-fn", target).CombinedOutput(); err != nil { + t.Fatalf("e2fsck: %v\n%s", err, out) + } +} + // TestCustomizeRealArmbian adapts the real Armbian image with the real // payload. Opt-in: ARMBIAN_IMG=path/to/uncompressed.img, and `make -// payload` must have run (debugfs, e2fsck and the packages). +// payload` must have run (the packages). func TestCustomizeRealArmbian(t *testing.T) { src := os.Getenv("ARMBIAN_IMG") if src == "" { t.Skip("set ARMBIAN_IMG to the uncompressed Armbian image") } payload := os.DirFS("../makerui/payload") - tools := "../makerui/payload/mac" img := filepath.Join(t.TempDir(), "a.img") if out, err := exec.Command("cp", "-c", src, img).CombinedOutput(); err != nil { if out, err := exec.Command("cp", src, img).CombinedOutput(); err != nil { @@ -70,17 +83,27 @@ func TestCustomizeRealArmbian(t *testing.T) { } s := testSettings() s.Board = "rock4cplus" - err := CustomizeArmbian(context.Background(), img, s, payload, tools, func(string, float64) {}) + err := CustomizeArmbian(context.Background(), img, s, payload, func(string, float64) {}) if err != nil { t.Fatal(err) } - d := debugfs{bin: filepath.Join(tools, "debugfs"), image: img} - shadow, _ := d.read(context.Background(), "/etc/shadow") + fsckRoot(t, img) + f, _ := os.Open(img) + r, err := openRoot(f) + if err != nil { + t.Fatal(err) + } + shadow, _ := r.read("/etc/shadow") if !strings.HasPrefix(string(shadow), "root:!") { t.Errorf("root not locked: %.40s", shadow) } + if st, err := r.fs.Stat("/etc/shadow"); err != nil || st.GID != 42 || st.Mode.Perm() != 0o640 { + t.Errorf("shadow: %+v %v", st, err) + } + f.Close() // Doing it again (a second build from the same image) must also work. - if err := CustomizeArmbian(context.Background(), img, s, payload, tools, func(string, float64) {}); err != nil { + if err := CustomizeArmbian(context.Background(), img, s, payload, func(string, float64) {}); err != nil { t.Fatalf("second run: %v", err) } + fsckRoot(t, img) } diff --git a/internal/maker/update.go b/internal/maker/update.go index 6ddfe14..505bc7b 100644 --- a/internal/maker/update.go +++ b/internal/maker/update.go @@ -1,13 +1,11 @@ package maker import ( - "context" "encoding/binary" "errors" "fmt" "io/fs" "os" - "path" "path/filepath" "strings" ) @@ -163,9 +161,9 @@ func updatePiBoot(dir string, all fs.FS) error { // armbianDirs are where Squirrel lives on an Armbian card, newest first. var armbianDirs = []string{armbianDir, "/boot/teslausb"} -func armbianFolder(ctx context.Context, d debugfs) (string, error) { +func armbianFolder(r *root) (string, error) { for _, dir := range armbianDirs { - if d.exists(ctx, dir+"/firstboot.sh") { + if r.exists(dir + "/firstboot.sh") { return dir, nil } } @@ -173,52 +171,45 @@ func armbianFolder(ctx context.Context, d debugfs) (string, error) { } // inspectArmbian looks at an Armbian card's root. -func inspectArmbian(ctx context.Context, d debugfs) (CardInfo, error) { - release, err := d.read(ctx, "/etc/armbian-release") +func inspectArmbian(r *root) (CardInfo, error) { + release, err := r.read("/etc/armbian-release") if err != nil || !strings.Contains(string(release), "BOARD=rockpi-4cplus") { return CardInfo{}, errors.New("this does not look like Armbian for the ROCK 4C+") } - dir, err := armbianFolder(ctx, d) + dir, err := armbianFolder(r) if err != nil { return CardInfo{}, err } - v, _ := d.read(ctx, dir+"/version") - return CardInfo{Board: "rock4cplus", Version: strings.TrimSpace(string(v)), Pending: !d.exists(ctx, dir+"/done")}, nil + v, _ := r.read(dir + "/version") + return CardInfo{Board: "rock4cplus", Version: strings.TrimSpace(string(v)), Pending: !r.exists(dir + "/done")}, nil } // updateArmbian puts the new Squirrel on an Armbian card's root. -func updateArmbian(ctx context.Context, d debugfs, all fs.FS, report Progress) error { - if _, err := inspectArmbian(ctx, d); err != nil { +func updateArmbian(r *root, all fs.FS, report Progress) error { + if _, err := inspectArmbian(r); err != nil { return err } - dir, err := armbianFolder(ctx, d) + dir, err := armbianFolder(r) if err != nil { return err } - armbianDir := dir current := dir == armbianDirs[0] - if out, err := d.tool(ctx, d.e2fsck, "-fn"); err != nil { - return fmt.Errorf("the card's file system has errors; not changing it: %s", lastLine(out)) - } files, err := armbianProgramIn(all, dir, current) if err != nil { return err } - remove := []string{armbianDir + "/done"} + remove := []string{dir + "/done"} for _, f := range legacyOld { - remove = append(remove, armbianDir+"/"+f) - } - out, _ := d.run(ctx, "ls -p "+armbianDir+"/debs") - for _, line := range strings.Split(string(out), "\n") { - // ls -p prints /inode/mode/uid/gid/name/size/. - f := strings.Split(line, "/") - if len(f) > 5 && strings.HasSuffix(f[5], ".deb") { - p := armbianDir + "/debs/" + path.Base(f[5]) - if _, ok := files[p]; !ok { - remove = append(remove, p) - } + remove = append(remove, dir+"/"+f) + } + // Packages of the old version that the new one does not bring. + old, _ := r.fs.ReadDir(dir + "/debs") + for _, e := range old { + p := dir + "/debs/" + e.Name + if _, ok := files[p]; !ok && strings.HasSuffix(e.Name, ".deb") && !e.IsDir() { + remove = append(remove, p) } } report("Updating the card", -1) - return d.put(ctx, files, remove, []string{armbianDir + "/debs"}, current, report) + return r.put(files, remove, []string{dir + "/debs"}, current, report) } diff --git a/internal/maker/update_darwin.go b/internal/maker/update_darwin.go index 6bd22de..6baaebc 100644 --- a/internal/maker/update_darwin.go +++ b/internal/maker/update_darwin.go @@ -4,7 +4,6 @@ import ( "context" "errors" "io/fs" - "path/filepath" ) // partitions returns the partitions of a disk and what diskutil says they @@ -84,8 +83,8 @@ func InspectCard(ctx context.Context, id string) (CardInfo, error) { } // UpdateCard puts the Squirrel of this app on a card that already holds -// it. all is the whole payload; tools holds debugfs and e2fsck. -func UpdateCard(ctx context.Context, id string, all fs.FS, tools func() (string, error), report Progress) (CardInfo, error) { +// it. all is the whole payload. +func UpdateCard(ctx context.Context, id string, all fs.FS, report Progress) (CardInfo, error) { d, err := checkDisk(ctx, id) if err != nil { return CardInfo{}, err @@ -95,10 +94,6 @@ func UpdateCard(ctx context.Context, id string, all fs.FS, tools func() (string, return CardInfo{}, err } if isArmbianCard(contents) { - dir, err := tools() - if err != nil { - return CardInfo{}, err - } report("Preparing the SD card", -1) if _, err := command(ctx, "diskutil", "unmountDisk", "/dev/"+id); err != nil { return CardInfo{}, err @@ -109,11 +104,14 @@ func UpdateCard(ctx context.Context, id string, all fs.FS, tools func() (string, return CardInfo{}, err } defer dev.Close() - dfs := debugfs{bin: filepath.Join(dir, "debugfs"), e2fsck: filepath.Join(dir, "e2fsck"), dev: dev} - if err := updateArmbian(ctx, dfs, all, report); err != nil { + r, err := openRoot(dev) + if err != nil { + return CardInfo{}, err + } + if err := updateArmbian(r, all, report); err != nil { return CardInfo{}, err } - info, err := inspectArmbian(ctx, dfs) + info, err := inspectArmbian(r) info.Disk = d dev.Close() report("Ejecting the card", -1) diff --git a/internal/maker/update_other.go b/internal/maker/update_other.go index 2bea7dc..2666b06 100644 --- a/internal/maker/update_other.go +++ b/internal/maker/update_other.go @@ -12,6 +12,6 @@ func InspectCard(ctx context.Context, id string) (CardInfo, error) { return CardInfo{}, errors.New("cards can only be read on macOS for now") } -func UpdateCard(ctx context.Context, id string, all fs.FS, tools func() (string, error), report Progress) (CardInfo, error) { +func UpdateCard(ctx context.Context, id string, all fs.FS, report Progress) (CardInfo, error) { return CardInfo{}, errors.New("cards can only be updated on macOS for now") } diff --git a/internal/maker/update_test.go b/internal/maker/update_test.go index 6dc6966..fcc78e0 100644 --- a/internal/maker/update_test.go +++ b/internal/maker/update_test.go @@ -71,7 +71,7 @@ func TestUpdatePiBoot(t *testing.T) { } } -// Updates a card made from the real Armbian image, through /dev/fd/3 as +// Updates a card made from the real Armbian image, through an open file as // on a real card. Opt-in like TestCustomizeRealArmbian. func TestUpdateRealArmbian(t *testing.T) { src := os.Getenv("ARMBIAN_IMG") @@ -79,7 +79,6 @@ func TestUpdateRealArmbian(t *testing.T) { t.Skip("set ARMBIAN_IMG to the uncompressed Armbian image") } payload := os.DirFS("../makerui/payload") - tools := "../makerui/payload/mac" img := filepath.Join(t.TempDir(), "a.img") if out, err := exec.Command("cp", src, img).CombinedOutput(); err != nil { t.Fatalf("copy: %v %s", err, out) @@ -87,7 +86,7 @@ func TestUpdateRealArmbian(t *testing.T) { s := testSettings() s.Board = "rock4cplus" ctx := context.Background() - if err := CustomizeArmbian(ctx, img, s, payload, tools, func(string, float64) {}); err != nil { + if err := CustomizeArmbian(ctx, img, s, payload, func(string, float64) {}); err != nil { t.Fatal(err) } f, err := os.OpenFile(img, os.O_RDWR, 0) @@ -95,30 +94,40 @@ func TestUpdateRealArmbian(t *testing.T) { t.Fatal(err) } defer f.Close() - d := debugfs{bin: filepath.Join(tools, "debugfs"), e2fsck: filepath.Join(tools, "e2fsck"), dev: f} // As after a first start: done, and a package that is no longer bundled. - tmp := filepath.Join(t.TempDir(), "x") - os.WriteFile(tmp, nil, 0o644) - if _, err := d.tool(ctx, d.bin, "-w", "-R", "write "+tmp+" "+armbianDir+"/done"); err != nil { + r, err := openRoot(f) + if err != nil { t.Fatal(err) } - if _, err := d.tool(ctx, d.bin, "-w", "-R", "write "+tmp+" "+armbianDir+"/debs/old_1.0_arm64.deb"); err != nil { + for _, p := range []string{armbianDir + "/done", armbianDir + "/debs/old_1.0_arm64.deb"} { + if err := r.fs.WriteFile(p, nil, 0o644, 0, 0); err != nil { + t.Fatal(err) + } + } + if err := r.fs.Flush(); err != nil { t.Fatal(err) } - if info, err := inspectArmbian(ctx, d); err != nil || info.Pending { + if r, err = openRoot(f); err != nil { + t.Fatal(err) + } + if info, err := inspectArmbian(r); err != nil || info.Pending { t.Fatalf("before: %+v %v", info, err) } - if err := updateArmbian(ctx, d, payload, func(string, float64) {}); err != nil { + if err := updateArmbian(r, payload, func(string, float64) {}); err != nil { + t.Fatal(err) + } + fsckRoot(t, img) + if r, err = openRoot(f); err != nil { t.Fatal(err) } - info, err := inspectArmbian(ctx, d) + info, err := inspectArmbian(r) if err != nil || !info.Pending { t.Errorf("after: %+v %v", info, err) } - if d.exists(ctx, armbianDir+"/debs/old_1.0_arm64.deb") { + if r.exists(armbianDir + "/debs/old_1.0_arm64.deb") { t.Error("old package left behind") } - if !d.exists(ctx, armbianDir+"/setup.env") { + if !r.exists(armbianDir + "/setup.env") { t.Error("settings removed") } } diff --git a/internal/makerui/makerui.go b/internal/makerui/makerui.go index b44d24a..c300905 100644 --- a/internal/makerui/makerui.go +++ b/internal/makerui/makerui.go @@ -343,10 +343,7 @@ func (a *Server) runBuild(ctx context.Context, s maker.Settings, report maker.Pr err = maker.Customize(ctx, out, s, payload, report) } } else { - var tools string - if tools, err = a.tools(); err == nil { - err = maker.CustomizeArmbian(ctx, out, s, a.all, tools, report) - } + err = maker.CustomizeArmbian(ctx, out, s, a.all, report) } if err != nil { os.Remove(out) @@ -355,29 +352,6 @@ func (a *Server) runBuild(ctx context.Context, s maker.Settings, report maker.Pr return out, nil } -// tools unpacks debugfs and e2fsck, which the app carries for Armbian's -// ext4 root, into the cache. -func (a *Server) tools() (string, error) { - dir := filepath.Join(a.cacheDir, "tools-"+a.opts.Version) - if err := os.MkdirAll(dir, 0o755); err != nil { - return "", err - } - for _, name := range []string{"debugfs", "e2fsck"} { - data, err := fs.ReadFile(a.all, "mac/"+name) - if err != nil { - return "", fmt.Errorf("this build of squirrel-maker lacks %s; build it with `make maker`", name) - } - tmp := filepath.Join(dir, name+".new") - if err := os.WriteFile(tmp, data, 0o755); err != nil { - return "", err - } - if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil { - return "", err - } - } - return dir, nil -} - // releases lists the OS images for a board: the tested one, then newer // ones from the publisher. Offline it is just the tested one. func (a *Server) releases(w http.ResponseWriter, r *http.Request) { @@ -469,7 +443,7 @@ func (a *Server) updateCard(w http.ResponseWriter, r *http.Request) { } go func() { report := func(step string, frac float64) { a.update(j, func(j *job) { j.Step, j.Frac = step, frac }) } - info, err := maker.UpdateCard(context.Background(), in.Disk, a.all, a.tools, report) + info, err := maker.UpdateCard(context.Background(), in.Disk, a.all, report) a.update(j, func(j *job) { j.Done = true if err != nil { diff --git a/internal/makerui/payload/README.md b/internal/makerui/payload/README.md index 80a5155..9a12451 100644 --- a/internal/makerui/payload/README.md +++ b/internal/makerui/payload/README.md @@ -6,8 +6,9 @@ not in git): - `files/`: the arm64 squirreld, the install scripts and the xfsprogs packages; on a Raspberry Pi card this is the `squirrel` folder on the boot partition. +- `files-armv6/`: the same for 32-bit Raspberry Pi OS on the original Pi + Zero W: an ARMv6 squirreld and Raspbian's packages. - `armbian/`: for the Radxa ROCK 4C+ (Armbian): NetworkManager, xfsprogs, exfatprogs and their dependencies, the USB overlay and the first-start unit. -- `mac/`: `debugfs` and `e2fsck` from e2fsprogs (GPL-2), built for Apple - silicon and Intel; the app writes Armbian's ext4 root with them. + From 5dc809e6ad9b44f419e0ff7a4b7ee653c765df3d Mon Sep 17 00:00:00 2001 From: Marcus Aram <4435760+marcusaram@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:16:32 +0100 Subject: [PATCH 2/8] Maker: customize Raspberry Pi OS images in Go, without hdiutil The fatfs package can now change a FAT32 volume: write and replace files and make directories, with long names and unique 8.3 aliases, every FAT copy and the FSInfo free count kept up to date. Customize uses it on the image file instead of having macOS mount it, so it runs the same on macOS and Windows. Tested in make test-linux: a volume made by mkfs.fat, with files Linux put there first, changed and judged by fsck.fat and read back by the kernel; and, opt-in, a real Raspberry Pi OS image customized and its boot partition judged by fsck.fat. --- Makefile | 2 +- internal/fatfs/fat32.go | 7 + internal/fatfs/fat32write.go | 510 ++++++++++++++++++ internal/fatfs/fat32write_test.go | 127 +++++ internal/fatfs/volume.go | 8 + internal/maker/customize.go | 100 ++++ internal/maker/customize_darwin.go | 113 ---- internal/maker/customize_other.go | 13 - ...omize_darwin_test.go => customize_test.go} | 13 + internal/maker/plist_darwin.go | 24 + 10 files changed, 790 insertions(+), 127 deletions(-) create mode 100644 internal/fatfs/fat32write.go create mode 100644 internal/fatfs/fat32write_test.go create mode 100644 internal/maker/customize.go delete mode 100644 internal/maker/customize_darwin.go delete mode 100644 internal/maker/customize_other.go rename internal/maker/{customize_darwin_test.go => customize_test.go} (77%) create mode 100644 internal/maker/plist_darwin.go diff --git a/Makefile b/Makefile index 6677942..20848b8 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ test-linux: mkdir -p dist/test for p in fatfs preserve gadget archive driveimage nbd ext4; do GOOS=linux go test -c -o dist/test/$$p.test ./internal/$$p; done docker run --rm --privileged -v "$(CURDIR):/src:ro" debian:trixie-slim sh -c '\ - apt-get update -qq >/dev/null && apt-get install -y -qq xfsprogs exfatprogs e2fsprogs >/dev/null 2>&1; \ + apt-get update -qq >/dev/null && apt-get install -y -qq xfsprogs exfatprogs e2fsprogs dosfstools >/dev/null 2>&1; \ truncate -s 1G /x.img && mkfs.xfs -q /x.img && mkdir /xfs && mount -o loop /x.img /xfs && \ cd /src/internal/fatfs && /src/dist/test/fatfs.test && \ cd /src/internal/gadget && /src/dist/test/gadget.test -test.v -test.run Mounted && \ diff --git a/internal/fatfs/fat32.go b/internal/fatfs/fat32.go index 1855be0..332dae8 100644 --- a/internal/fatfs/fat32.go +++ b/internal/fatfs/fat32.go @@ -43,6 +43,13 @@ func openFAT32(r io.ReaderAt, off, size int64, bs []byte) (*Volume, error) { rootCluster: binary.LittleEndian.Uint32(bs[44:]), eoc: 0x0FFFFFF7, } + v.fatStart, v.fatSize, v.numFATs = v.fatOffset, fatsz*bps, nfats + if fi := int64(binary.LittleEndian.Uint16(bs[48:])); fi > 0 && fi < rsvd { + sig := make([]byte, 4) + if _, err := r.ReadAt(sig, off+fi*bps); err == nil && binary.LittleEndian.Uint32(sig) == 0x41615252 { + v.fsInfo = off + fi*bps + } + } if ext := binary.LittleEndian.Uint16(bs[40:]); ext&0x80 != 0 { // FAT mirroring disabled; use the active FAT v.fatOffset += int64(ext&0x0F) * fatsz * bps diff --git a/internal/fatfs/fat32write.go b/internal/fatfs/fat32write.go new file mode 100644 index 0000000..1031829 --- /dev/null +++ b/internal/fatfs/fat32write.go @@ -0,0 +1,510 @@ +package fatfs + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "path" + "strings" + "time" + "unicode/utf16" +) + +// Writing FAT32: just what Squirrel Maker needs on Raspberry Pi OS's boot +// partition in an image: write and replace files and make directories. +// New names get a long name and a unique 8.3 alias, as Windows and Linux +// give them; every FAT copy and the FSInfo free count are kept up to date. + +// ReadWriterAt is what OpenRW changes the volume on. +type ReadWriterAt interface { + io.ReaderAt + io.WriterAt +} + +// OpenRW is Open for changing a FAT32 volume. +func OpenRW(rw ReadWriterAt, size int64) (*Volume, error) { + v, err := Open(rw, size) + if err != nil { + return nil, err + } + if v.Type != "fat32" { + return nil, fmt.Errorf("fatfs: changing %s is not supported", v.Type) + } + v.w = rw + return v, nil +} + +const ( + attrDir = 0x10 + attrArchive = 0x20 + attrLFN = 0x0F + fatEOC = 0x0FFFFFFF +) + +// loadFAT reads the active FAT once; changes go to it and to every copy +// on the volume. +func (v *Volume) loadFAT() error { + if v.fat != nil { + return nil + } + fat := make([]byte, v.fatSize) + if _, err := v.r.ReadAt(fat, v.fatOffset); err != nil { + return err + } + v.fat = fat + return nil +} + +func (v *Volume) setFAT(c, val uint32) error { + if err := v.loadFAT(); err != nil { + return err + } + o := int64(c) * 4 + old := binary.LittleEndian.Uint32(v.fat[o:]) + binary.LittleEndian.PutUint32(v.fat[o:], old&0xF0000000|val&0x0FFFFFFF) + for i := int64(0); i < v.numFATs; i++ { + if _, err := v.w.WriteAt(v.fat[o:o+4], v.fatStart+i*v.fatSize+o); err != nil { + return err + } + } + return nil +} + +func (v *Volume) getFAT(c uint32) uint32 { + return binary.LittleEndian.Uint32(v.fat[int64(c)*4:]) & 0x0FFFFFFF +} + +// adjustFree changes the FSInfo free cluster count, when it is kept. +func (v *Volume) adjustFree(d int64) error { + if v.fsInfo == 0 { + return nil + } + var b [4]byte + if _, err := v.r.ReadAt(b[:], v.fsInfo+488); err != nil { + return err + } + n := binary.LittleEndian.Uint32(b[:]) + if n == 0xFFFFFFFF { + return nil // not kept + } + binary.LittleEndian.PutUint32(b[:], uint32(int64(n)+d)) + _, err := v.w.WriteAt(b[:], v.fsInfo+488) + return err +} + +// alloc takes n free clusters, links them into a chain, and zeroes them. +func (v *Volume) alloc(n int) ([]uint32, error) { + if err := v.loadFAT(); err != nil { + return nil, err + } + var got []uint32 + for c := uint32(2); c < v.clusterCount+2 && len(got) < n; c++ { + if v.getFAT(c) == 0 { + got = append(got, c) + } + } + if len(got) < n { + return nil, errors.New("fatfs: the volume is full") + } + for i, c := range got { + next := uint32(fatEOC) + if i+1 < len(got) { + next = got[i+1] + } + if err := v.setFAT(c, next); err != nil { + return nil, err + } + } + return got, v.adjustFree(-int64(n)) +} + +func (v *Volume) freeChain(first uint32) error { + if first < 2 { + return nil + } + if err := v.loadFAT(); err != nil { + return err + } + freed := int64(0) + for c := first; c >= 2 && c < v.clusterCount+2; { + next := v.getFAT(c) + if err := v.setFAT(c, 0); err != nil { + return err + } + freed++ + if next >= v.eoc || freed > int64(v.clusterCount) { + break + } + c = next + } + return v.adjustFree(freed) +} + +// writeClusters writes data over a chain of clusters, zero filled. +func (v *Volume) writeClusters(chain []uint32, data []byte) error { + buf := make([]byte, v.ClusterSize) + for i, c := range chain { + clear(buf) + if lo := int64(i) * v.ClusterSize; lo < int64(len(data)) { + copy(buf, data[lo:]) + } + off, err := v.clusterOffset(c) + if err != nil { + return err + } + if _, err := v.w.WriteAt(buf, off); err != nil { + return err + } + } + return nil +} + +// dir is a directory's raw entries and the clusters they are in. +type dir struct { + first uint32 + clusters []uint32 + data []byte +} + +func (v *Volume) loadDir(first uint32) (*dir, error) { + runs, err := v.chain(first, 0, false) + if err != nil { + return nil, err + } + d := &dir{first: first} + for _, r := range runs { + for i := uint32(0); i < r[1]; i++ { + d.clusters = append(d.clusters, r[0]+i) + } + } + if d.data, err = v.readDirData(first, 0, false); err != nil { + return nil, err + } + return d, nil +} + +func (v *Volume) slotOffset(d *dir, slot int) (int64, error) { + at := int64(slot) * 32 + c := at / v.ClusterSize + if c >= int64(len(d.clusters)) { + return 0, fmt.Errorf("%w: directory slot %d", ErrCorrupt, slot) + } + off, err := v.clusterOffset(d.clusters[c]) + return off + at%v.ClusterSize, err +} + +func (v *Volume) writeSlot(d *dir, slot int, e []byte) error { + copy(d.data[slot*32:], e) + off, err := v.slotOffset(d, slot) + if err != nil { + return err + } + _, err = v.w.WriteAt(e, off) + return err +} + +// find returns the slot of name's short entry in d, case-insensitively. +func (v *Volume) find(d *dir, name string) (int, bool) { + var lfn []uint16 + var sum byte + for i := 0; i+32 <= len(d.data); i += 32 { + e := d.data[i : i+32] + if e[0] == 0x00 { + break + } + if e[0] == 0xE5 { + lfn = nil + continue + } + if e[11]&0x3F == attrLFN { + ord := int(e[0] & 0x1F) + if e[0]&0x40 != 0 { + lfn, sum = make([]uint16, ord*13), e[13] + } + if lfn == nil || ord == 0 || ord*13 > len(lfn) { + lfn = nil + continue + } + p := (ord - 1) * 13 + for _, r := range [][2]int{{1, 11}, {14, 26}, {28, 32}} { + for k := r[0]; k < r[1]; k += 2 { + lfn[p] = binary.LittleEndian.Uint16(e[k:]) + p++ + } + } + continue + } + if e[11]&0x08 != 0 { + lfn = nil + continue + } + n := shortName(e) + if lfn != nil && sfnChecksum(e[:11]) == sum { + n = decodeLFN(lfn) + } + lfn = nil + if strings.EqualFold(n, name) || strings.EqualFold(shortName(e), name) { + return i / 32, true + } + } + return 0, false +} + +// alias makes a unique 8.3 name for name in d: BASE~N.EXT. +func alias(d *dir, name string) ([11]byte, error) { + clean := func(s string, n int) string { + var b strings.Builder + for _, r := range strings.ToUpper(s) { + if b.Len() >= n { + break + } + switch { + case r >= 'A' && r <= 'Z', r >= '0' && r <= '9', strings.ContainsRune("$%'-_@~`!(){}^#&", r): + b.WriteRune(r) + case r == ' ' || r == '.': + default: + b.WriteRune('_') + } + } + return b.String() + } + base, ext := name, "" + if i := strings.LastIndex(name, "."); i > 0 { + base, ext = name[:i], name[i+1:] + } + b, x := clean(base, 8), clean(ext, 3) + if b == "" { + b = "FILE" + } + taken := map[string]bool{} + for i := 0; i+32 <= len(d.data); i += 32 { + e := d.data[i : i+32] + if e[0] == 0x00 { + break + } + if e[0] != 0xE5 && e[11]&0x3F != attrLFN { + taken[string(e[:11])] = true + } + } + for n := 1; n < 1000000; n++ { + tail := fmt.Sprintf("~%d", n) + var sfn [11]byte + copy(sfn[:], fmt.Sprintf("%-8s%-3s", b[:min(len(b), 8-len(tail))]+tail, x)) + if !taken[string(sfn[:])] { + return sfn, nil + } + } + return [11]byte{}, errors.New("fatfs: no free short name") +} + +func dosStamp(t time.Time) (date, tm uint16) { + if t.Year() < 1980 { + t = time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC) + } + date = uint16((t.Year()-1980)<<9 | int(t.Month())<<5 | t.Day()) + tm = uint16(t.Hour()<<11 | t.Minute()<<5 | t.Second()/2) + return +} + +func shortEntry(sfn [11]byte, attr byte, cluster uint32, size uint32, t time.Time) []byte { + e := make([]byte, 32) + copy(e, sfn[:]) + e[11] = attr + date, tm := dosStamp(t) + binary.LittleEndian.PutUint16(e[14:], tm) + binary.LittleEndian.PutUint16(e[16:], date) + binary.LittleEndian.PutUint16(e[18:], date) + binary.LittleEndian.PutUint16(e[20:], uint16(cluster>>16)) + binary.LittleEndian.PutUint16(e[22:], tm) + binary.LittleEndian.PutUint16(e[24:], date) + binary.LittleEndian.PutUint16(e[26:], uint16(cluster)) + binary.LittleEndian.PutUint32(e[28:], size) + return e +} + +// lfnEntries are the long name entries for name, in on-disk order. +func lfnEntries(name string, sum byte) [][]byte { + u := utf16.Encode([]rune(name)) + n := (len(u) + 12) / 13 + padded := make([]uint16, n*13) + for i := range padded { + switch { + case i < len(u): + padded[i] = u[i] + case i == len(u): + padded[i] = 0 + default: + padded[i] = 0xFFFF + } + } + var out [][]byte + for ord := n; ord >= 1; ord-- { + e := make([]byte, 32) + e[0] = byte(ord) + if ord == n { + e[0] |= 0x40 + } + e[11], e[13] = attrLFN, sum + p := (ord - 1) * 13 + for _, r := range [][2]int{{1, 11}, {14, 26}, {28, 32}} { + for k := r[0]; k < r[1]; k += 2 { + binary.LittleEndian.PutUint16(e[k:], padded[p]) + p++ + } + } + out = append(out, e) + } + return out +} + +// add puts a new name with its short entry into d, in free slots or a +// new cluster. +func (v *Volume) add(d *dir, name string, attr byte, cluster, size uint32, t time.Time) error { + if name == "" || len(name) > 255 || strings.ContainsAny(name, `/\:*?"<>|`) { + return fmt.Errorf("fatfs: name %q", name) + } + sfn, err := alias(d, name) + if err != nil { + return err + } + entries := append(lfnEntries(name, sfnChecksum(sfn[:])), shortEntry(sfn, attr, cluster, size, t)) + need := len(entries) + start, run := -1, 0 + for i := 0; i+32 <= len(d.data); i += 32 { + if b := d.data[i]; b == 0x00 || b == 0xE5 { + if run == 0 { + start = i / 32 + } + run++ + if run == need { + break + } + } else { + run = 0 + } + } + if run < need { + // Grow the directory by a cluster. + c, err := v.alloc(1) + if err != nil { + return err + } + if err := v.writeClusters(c, nil); err != nil { + return err + } + if err := v.setFAT(d.clusters[len(d.clusters)-1], c[0]); err != nil { + return err + } + if run == 0 { + start = len(d.data) / 32 + } + d.clusters = append(d.clusters, c[0]) + d.data = append(d.data, make([]byte, v.ClusterSize)...) + } + for i, e := range entries { + if err := v.writeSlot(d, start+i, e); err != nil { + return err + } + } + return nil +} + +// lookupDir follows a path of directories from the root. +func (v *Volume) lookupDir(p string) (*dir, error) { + cur := v.rootCluster + for _, part := range strings.Split(strings.Trim(path.Clean("/"+p), "/"), "/") { + if part == "" { + continue + } + d, err := v.loadDir(cur) + if err != nil { + return nil, err + } + slot, ok := v.find(d, part) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrNotFound, p) + } + e := d.data[slot*32:] + if e[11]&attrDir == 0 { + return nil, fmt.Errorf("fatfs: %s is not a directory", part) + } + cur = uint32(binary.LittleEndian.Uint16(e[20:]))<<16 | uint32(binary.LittleEndian.Uint16(e[26:])) + } + return v.loadDir(cur) +} + +// Mkdir makes a directory whose parent exists; one that is there already +// is fine. +func (v *Volume) Mkdir(p string, t time.Time) error { + p = path.Clean("/" + p) + parent, err := v.lookupDir(path.Dir(p)) + if err != nil { + return err + } + if slot, ok := v.find(parent, path.Base(p)); ok { + if parent.data[slot*32+11]&attrDir != 0 { + return nil + } + return fmt.Errorf("fatfs: %s exists and is not a directory", p) + } + c, err := v.alloc(1) + if err != nil { + return err + } + dotdot := parent.first + if dotdot == v.rootCluster { + dotdot = 0 + } + body := make([]byte, v.ClusterSize) + copy(body, shortEntry([11]byte{'.', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '}, attrDir, c[0], 0, t)) + copy(body[32:], shortEntry([11]byte{'.', '.', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '}, attrDir, dotdot, 0, t)) + if err := v.writeClusters(c, body); err != nil { + return err + } + return v.add(parent, path.Base(p), attrDir, c[0], 0, t) +} + +// WriteFile writes a file whose directory exists, replacing one that is +// there (in its directory entry). +func (v *Volume) WriteFile(p string, data []byte, t time.Time) error { + if int64(len(data)) > 0xFFFFFFFF { + return errors.New("fatfs: file too large for FAT32") + } + p = path.Clean("/" + p) + d, err := v.lookupDir(path.Dir(p)) + if err != nil { + return err + } + n := int((int64(len(data)) + v.ClusterSize - 1) / v.ClusterSize) + var chain []uint32 + if n > 0 { + if chain, err = v.alloc(n); err != nil { + return err + } + if err := v.writeClusters(chain, data); err != nil { + return err + } + } + first := uint32(0) + if len(chain) > 0 { + first = chain[0] + } + if slot, ok := v.find(d, path.Base(p)); ok { + e := append([]byte(nil), d.data[slot*32:slot*32+32]...) + if e[11]&attrDir != 0 { + return fmt.Errorf("fatfs: %s is a directory", p) + } + old := uint32(binary.LittleEndian.Uint16(e[20:]))<<16 | uint32(binary.LittleEndian.Uint16(e[26:])) + var sfn [11]byte + copy(sfn[:], e[:11]) + ne := shortEntry(sfn, e[11]|attrArchive, first, uint32(len(data)), t) + ne[12] = e[12] // keep the lower case flags of a plain 8.3 name + copy(ne[13:20], e[13:20]) + if err := v.writeSlot(d, slot, ne); err != nil { + return err + } + return v.freeChain(old) + } + return v.add(d, path.Base(p), attrArchive, first, uint32(len(data)), t) +} diff --git a/internal/fatfs/fat32write_test.go b/internal/fatfs/fat32write_test.go new file mode 100644 index 0000000..da85e58 --- /dev/null +++ b/internal/fatfs/fat32write_test.go @@ -0,0 +1,127 @@ +package fatfs + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// Changes a FAT32 volume made by mkfs.fat, with files Linux put there +// first, then lets fsck.fat judge it and, as root, Linux read it back. +// Needs dosfstools; runs in `make test-linux`. +func TestFAT32Write(t *testing.T) { + for _, tool := range []string{"mkfs.fat", "fsck.fat"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("no %s", tool) + } + } + sh := func(name string, args ...string) { + t.Helper() + if out, err := exec.Command(name, args...).CombinedOutput(); err != nil { + t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out) + } + } + img := filepath.Join(t.TempDir(), "boot.img") + sh("truncate", "-s", "128M", img) + sh("mkfs.fat", "-F", "32", "-n", "BOOTFS", img) + root := os.Getuid() == 0 + if root { + // What Raspberry Pi OS has there before the maker comes. + mnt := t.TempDir() + sh("mount", "-o", "loop", img, mnt) + os.WriteFile(filepath.Join(mnt, "cmdline.txt"), []byte("console=serial0,115200 root=PARTUUID=x rootwait quiet init=/usr/lib/raspberrypi-sys-mods/firstboot\n"), 0o644) + os.WriteFile(filepath.Join(mnt, "config.txt"), []byte("[all]\ndtoverlay=vc4-kms-v3d\n"), 0o644) + os.MkdirAll(filepath.Join(mnt, "overlays"), 0o755) + for i := 0; i < 150; i++ { + os.WriteFile(filepath.Join(mnt, "overlays", fmt.Sprintf("overlay-number-%03d.dtbo", i)), []byte{byte(i)}, 0o644) + } + sh("umount", mnt) + } + + f, err := os.OpenFile(img, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + fi, _ := f.Stat() + v, err := OpenRW(f, fi.Size()) + if err != nil { + t.Fatal(err) + } + now := time.Now() + want := map[string][]byte{ + "cmdline.txt": []byte("console=serial0,115200 root=PARTUUID=x rootwait quiet cfg80211.ieee80211_regdom=NL\n"), + "config.txt": bytes.Repeat([]byte("[all]\ndtoverlay=dwc2\n"), 400), + "user-data": []byte("#cloud-config\nhostname: squirrel\n"), + "network-config": []byte("network:\n version: 2\n"), + "ssh": {}, + "squirrel/squirreld": bytes.Repeat([]byte("squirreld!"), 900_000), + "squirrel/squirrel-firstboot.service": []byte("[Unit]\n"), + "squirrel/debs/xfsprogs_6.13.0-2+deb13u1_arm64.deb": bytes.Repeat([]byte{1, 2, 3}, 350_000), + "squirrel/debs/libinih1_59-1_arm64.deb": bytes.Repeat([]byte{4}, 5000), + } + for i := 0; i < 120; i++ { + want[fmt.Sprintf("many/a file with spaces and a long name %03d.txt", i)] = []byte(fmt.Sprint(i)) + } + for _, d := range []string{"squirrel", "squirrel/debs", "many"} { + if err := v.Mkdir(d, now); err != nil { + t.Fatal(err) + } + } + if err := v.Mkdir("squirrel", now); err != nil { + t.Errorf("making an existing directory: %v", err) + } + for p, data := range want { + if err := v.WriteFile(p, data, now); err != nil { + t.Fatalf("%s: %v", p, err) + } + } + // Replacing again, with something smaller. + want["squirrel/squirreld"] = []byte("small now") + if err := v.WriteFile("squirrel/squirreld", want["squirrel/squirreld"], now); err != nil { + t.Fatal(err) + } + f.Close() + if out, err := exec.Command("fsck.fat", "-n", "-V", img).CombinedOutput(); err != nil { + t.Fatalf("fsck.fat found problems: %v\n%s", err, out) + } + + f, _ = os.Open(img) + defer f.Close() + r, err := Open(f, fi.Size()) + if err != nil { + t.Fatal(err) + } + for p, data := range want { + e, err := r.Lookup(p) + if err != nil { + t.Errorf("%s: %v", p, err) + continue + } + fr, _ := r.OpenFile(e) + got, _ := io.ReadAll(fr.NewReader()) + if !bytes.Equal(got, data) { + t.Errorf("%s: %d bytes, want %d", p, len(got), len(data)) + } + } + if !root { + return + } + mnt := t.TempDir() + sh("mount", "-o", "loop,ro", img, mnt) + defer exec.Command("umount", mnt).Run() + for p, data := range want { + got, err := os.ReadFile(filepath.Join(mnt, p)) + if err != nil || !bytes.Equal(got, data) { + t.Errorf("Linux reads %s: %d bytes, want %d: %v", p, len(got), len(data), err) + } + } + if list, _ := os.ReadDir(filepath.Join(mnt, "overlays")); len(list) != 150 { + t.Errorf("Linux lists %d overlays", len(list)) + } +} diff --git a/internal/fatfs/volume.go b/internal/fatfs/volume.go index 01980a4..fdf837f 100644 --- a/internal/fatfs/volume.go +++ b/internal/fatfs/volume.go @@ -61,6 +61,14 @@ type Volume struct { bitmap int64 // absolute offset of the exFAT allocation bitmap, 0 if unknown readDir func(dir Entry) ([]Entry, error) + + // For changing a FAT32 volume (OpenRW). + w io.WriterAt + fat []byte // the active FAT, read once + fatStart int64 // the first FAT copy + fatSize int64 // bytes per copy + numFATs int64 + fsInfo int64 // the FSInfo sector, 0 when there is none } var ( diff --git a/internal/maker/customize.go b/internal/maker/customize.go new file mode 100644 index 0000000..eb51ee6 --- /dev/null +++ b/internal/maker/customize.go @@ -0,0 +1,100 @@ +package maker + +import ( + "bytes" + "context" + "fmt" + "io/fs" + "os" + "path" + "sort" + "time" + + "github.com/marcusaram/squirrel/internal/fatfs" +) + +// Customize writes the settings and payload onto the boot partition of an +// uncompressed Raspberry Pi OS image, with the fatfs package: no mounting, +// no root rights and no tools, on macOS and Windows alike. +func Customize(ctx context.Context, image string, s Settings, payload fs.FS, report Progress) error { + if err := CheckLayout(image); err != nil { + return err + } + files, err := bootFiles(s, payload) + if err != nil { + return err + } + report("Adapting the image", -1) + cmdline, err := ReadBootFile(image, "cmdline.txt") + if err != nil { + return fmt.Errorf("this does not look like a Raspberry Pi boot partition: %w", err) + } + newCmdline, err := EditCmdline(string(cmdline), s.Country) + if err != nil { + return err + } + files["cmdline.txt"] = []byte(newCmdline) + cfgTxt, err := ReadBootFile(image, "config.txt") + if err != nil { + return err + } + files["config.txt"] = []byte(EditConfig(string(cfgTxt), s.SDOverclock)) + + f, err := os.OpenFile(image, os.O_RDWR, 0) + if err != nil { + return err + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return err + } + v, err := fatfs.OpenRW(f, fi.Size()) + if err != nil { + return err + } + names := make([]string, 0, len(files)) + dirs := map[string]bool{} + for name := range files { + names = append(names, name) + for d := path.Dir(name); d != "."; d = path.Dir(d) { + dirs[d] = true + } + } + // Parents before children. + var dirList []string + for d := range dirs { + dirList = append(dirList, d) + } + sort.Strings(dirList) + sort.Strings(names) + now := time.Now() + for _, d := range dirList { + if err := v.Mkdir(d, now); err != nil { + return fmt.Errorf("making %s: %w", d, err) + } + } + for _, name := range names { + if err := ctx.Err(); err != nil { + return err + } + if err := v.WriteFile(name, files[name], now); err != nil { + return fmt.Errorf("writing %s: %w", name, err) + } + } + if err := f.Sync(); err != nil { + return err + } + + // Read everything back through a fresh open. + for _, name := range names { + got, err := ReadBootFile(image, name) + if err != nil { + return fmt.Errorf("checking %s: %w", name, err) + } + if !bytes.Equal(got, files[name]) { + return fmt.Errorf("checking %s: content differs after writing", name) + } + } + return nil +} diff --git a/internal/maker/customize_darwin.go b/internal/maker/customize_darwin.go deleted file mode 100644 index 0a7ef16..0000000 --- a/internal/maker/customize_darwin.go +++ /dev/null @@ -1,113 +0,0 @@ -package maker - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/fs" - "os" - "os/exec" - "path/filepath" - "strings" -) - -// plist runs a macOS tool that prints a property list and decodes it. -func plist(ctx context.Context, v any, name string, args ...string) error { - out, err := command(ctx, name, args...) - if err != nil { - return err - } - conv := exec.CommandContext(ctx, "plutil", "-convert", "json", "-o", "-", "-") - conv.Stdin = bytes.NewReader(out) - js, err := conv.Output() - if err != nil { - return fmt.Errorf("plutil: %w", err) - } - return json.Unmarshal(js, v) -} - -// Customize writes the settings and payload onto the boot partition of an -// uncompressed Raspberry Pi OS image. macOS mounts the FAT partition -// itself, so no root rights or extra tools are needed. -func Customize(ctx context.Context, image string, s Settings, payload fs.FS, report Progress) error { - if err := CheckLayout(image); err != nil { - return err - } - files, err := bootFiles(s, payload) - if err != nil { - return err - } - report("Adapting the image", -1) - out, err := command(ctx, "hdiutil", "attach", "-imagekey", "diskimage-class=CRawDiskImage", "-nomount", image) - if err != nil { - return err - } - disk := strings.Fields(string(out))[0] // e.g. /dev/disk7 - defer command(context.Background(), "hdiutil", "detach", "-force", disk) - - if _, err := command(ctx, "diskutil", "mount", disk+"s1"); err != nil { - return err - } - var info struct{ MountPoint string } - if err := plist(ctx, &info, "diskutil", "info", "-plist", disk+"s1"); err != nil { - return err - } - if info.MountPoint == "" { - return fmt.Errorf("boot partition of %s did not mount", disk) - } - mp := info.MountPoint - if _, err := os.Stat(filepath.Join(mp, "cmdline.txt")); err != nil { - return fmt.Errorf("%s does not look like a Raspberry Pi boot partition", mp) - } - - cmdline, err := os.ReadFile(filepath.Join(mp, "cmdline.txt")) - if err != nil { - return err - } - newCmdline, err := EditCmdline(string(cmdline), s.Country) - if err != nil { - return err - } - files["cmdline.txt"] = []byte(newCmdline) - cfgTxt, err := os.ReadFile(filepath.Join(mp, "config.txt")) - if err != nil { - return err - } - files["config.txt"] = []byte(EditConfig(string(cfgTxt), s.SDOverclock)) - - for name, data := range files { - p := filepath.Join(mp, filepath.FromSlash(name)) - if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { - return err - } - if err := os.WriteFile(p, data, 0o644); err != nil { - return fmt.Errorf("writing %s: %w", name, err) - } - } - // macOS stores extended attributes on FAT as ._ files; the Pi does not - // need them. - exec.Command("dot_clean", "-m", mp).Run() - for _, junk := range []string{".fseventsd", ".Spotlight-V100", ".Trashes"} { - os.RemoveAll(filepath.Join(mp, junk)) - } - exec.Command("sync").Run() - if _, err := command(ctx, "diskutil", "unmount", disk+"s1"); err != nil { - return err - } - if _, err := command(ctx, "hdiutil", "detach", disk); err != nil { - return err - } - - // Read back through our own FAT reader, independent of macOS. - for _, name := range []string{"cmdline.txt", "user-data", "network-config", "squirrel/squirreld"} { - got, err := ReadBootFile(image, name) - if err != nil { - return fmt.Errorf("checking %s: %w", name, err) - } - if !bytes.Equal(got, files[name]) { - return fmt.Errorf("checking %s: content differs after writing", name) - } - } - return nil -} diff --git a/internal/maker/customize_other.go b/internal/maker/customize_other.go deleted file mode 100644 index c19bbc0..0000000 --- a/internal/maker/customize_other.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build !darwin - -package maker - -import ( - "context" - "errors" - "io/fs" -) - -func Customize(ctx context.Context, image string, s Settings, payload fs.FS, report Progress) error { - return errors.New("adapting the image is only supported on macOS for now") -} diff --git a/internal/maker/customize_darwin_test.go b/internal/maker/customize_test.go similarity index 77% rename from internal/maker/customize_darwin_test.go rename to internal/maker/customize_test.go index 9c74802..2fd1e88 100644 --- a/internal/maker/customize_darwin_test.go +++ b/internal/maker/customize_test.go @@ -2,8 +2,10 @@ package maker import ( "context" + "fmt" "io" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -66,4 +68,15 @@ func TestCustomizeRealImage(t *testing.T) { } return nil }) + // fsck.fat, where it is installed, judges the boot partition. + if _, err := exec.LookPath("fsck.fat"); err == nil { + part := filepath.Join(t.TempDir(), "boot.img") + if out, err := exec.Command("dd", "if="+img, "of="+part, fmt.Sprintf("bs=%d", 512), fmt.Sprintf("skip=%d", v.Offset/512), + fmt.Sprintf("count=%d", 1024*1024)).CombinedOutput(); err != nil { + t.Fatalf("dd: %v %s", err, out) + } + if out, err := exec.Command("fsck.fat", "-n", part).CombinedOutput(); err != nil { + t.Errorf("fsck.fat: %v\n%s", err, out) + } + } } diff --git a/internal/maker/plist_darwin.go b/internal/maker/plist_darwin.go new file mode 100644 index 0000000..34640c8 --- /dev/null +++ b/internal/maker/plist_darwin.go @@ -0,0 +1,24 @@ +package maker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" +) + +// plist runs a macOS tool that prints a property list and decodes it. +func plist(ctx context.Context, v any, name string, args ...string) error { + out, err := command(ctx, name, args...) + if err != nil { + return err + } + conv := exec.CommandContext(ctx, "plutil", "-convert", "json", "-o", "-", "-") + conv.Stdin = bytes.NewReader(out) + js, err := conv.Output() + if err != nil { + return fmt.Errorf("plutil: %w", err) + } + return json.Unmarshal(js, v) +} From 5d35af7fdf575512f97d94db8bbd25b839980988 Mon Sep 17 00:00:00 2001 From: Marcus Aram <4435760+marcusaram@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:25:44 +0100 Subject: [PATCH 3/8] Squirrel Maker for Windows - Cards on Windows: listed through Get-Disk (USB, SD and MMC disks, not the system disk, writable, 8 GB to 2 TB), cleared so Windows lets go of their volumes, written through \\.\PhysicalDriveN and read back. Updating finds a Raspberry Pi card's boot partition by its drive letter, and an Armbian card's ext4 root on the raw disk. - WriteRaw writes the first megabyte last, so no partition is mounted halfway, by sector offsets, which raw disks need on both systems. - The app (Wails, without cgo) builds for Windows and asks for administrator rights when it starts; the command line version says when it lacks them and opens the browser the Windows way. - The country comes from the system's region, the time zone, where the system has no IANA name for it, from the browser; the pages say computer instead of Mac, and Explorer instead of Finder on Windows. - make app-windows builds a zip with both programs. CI tests on Windows too, with a virtual disk as the card: listed, written, made a Raspberry Pi card, inspected and updated. Releases carry the zip. --- .github/workflows/ci.yml | 24 ++- .github/workflows/release.yml | 15 +- Makefile | 17 +++ README.md | 14 +- cmd/squirrel-app/elevate_darwin.go | 5 + cmd/squirrel-app/elevate_windows.go | 28 ++++ cmd/squirrel-app/main.go | 11 +- cmd/squirrel-maker/admin_other.go | 6 + cmd/squirrel-maker/admin_windows.go | 12 ++ cmd/squirrel-maker/main.go | 22 ++- internal/maker/disks_other.go | 2 +- internal/maker/disks_windows.go | 220 +++++++++++++++++++++++++++ internal/maker/disks_windows_test.go | 110 ++++++++++++++ internal/maker/update_other.go | 2 +- internal/maker/update_windows.go | 130 ++++++++++++++++ internal/maker/write.go | 72 +++++---- internal/maker/write_test.go | 45 ++++++ internal/makerui/country_darwin.go | 18 +++ internal/makerui/country_other.go | 6 + internal/makerui/country_windows.go | 22 +++ internal/makerui/makerui.go | 16 +- internal/makerui/static/i18n.js | 16 +- internal/makerui/static/index.html | 6 +- internal/makerui/static/maker.js | 9 +- 24 files changed, 767 insertions(+), 61 deletions(-) create mode 100644 cmd/squirrel-app/elevate_darwin.go create mode 100644 cmd/squirrel-app/elevate_windows.go create mode 100644 cmd/squirrel-maker/admin_other.go create mode 100644 cmd/squirrel-maker/admin_windows.go create mode 100644 internal/maker/disks_windows.go create mode 100644 internal/maker/disks_windows_test.go create mode 100644 internal/maker/update_windows.go create mode 100644 internal/maker/write_test.go create mode 100644 internal/makerui/country_darwin.go create mode 100644 internal/makerui/country_other.go create mode 100644 internal/makerui/country_windows.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8626f29..f1ca873 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ # Tests every push to main and every pull request: the Go tests on Linux -# (also on a real XFS filesystem, for reflinks) and on macOS (the maker's -# Mac code), formatting, the builds for every board, and the web pages' -# scripts. +# (also on real XFS, ext4 and FAT file systems), on macOS and on Windows +# (the maker's code for each, on Windows with a virtual disk as the card), +# formatting, the builds for every board, and the web pages' scripts. name: CI on: @@ -52,3 +52,21 @@ jobs: - run: make test - name: Squirrel Maker builds (app and command line, without the payload) run: make app-bundle maker-bin VERSION=ci + + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Tests of the maker's packages + run: go test ./internal/ext4 ./internal/fatfs ./internal/maker ./internal/makerui + - name: A virtual disk as the card + env: + SQUIRREL_TEST_VHD: ${{ runner.temp }}\card.vhdx + run: go test -v -run WindowsCard ./internal/maker + - name: Squirrel Maker builds (app and command line) + run: | + go build -o squirrel-app.exe ./cmd/squirrel-app + go build -o squirrel-maker.exe ./cmd/squirrel-maker diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 13a941f..b941615 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ # A release for every version tag (v1.2.3): squirreld for each board, and -# Squirrel Maker as a Mac app and as a command line program, with the -# packages a new card installs at its first start. +# Squirrel Maker for macOS (app and command line) and Windows (both in one +# zip), with the packages a new card installs at its first start. # # git tag v0.16.0 && git push origin v0.16.0 name: Release @@ -31,11 +31,18 @@ jobs: run: make release VERSION="$VERSION" - name: Packages for a new card run: make payload VERSION="$VERSION" + - name: Squirrel Maker for Windows + run: make app-windows-bundle VERSION="$VERSION" - uses: actions/upload-artifact@v4 with: name: squirreld path: dist/squirreld-*.tar.gz if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: maker-windows + path: dist/Squirrel-Maker-*-windows.zip + if-no-files-found: error - uses: actions/upload-artifact@v4 with: name: payload @@ -86,6 +93,10 @@ jobs: with: name: maker path: out + - uses: actions/download-artifact@v4 + with: + name: maker-windows + path: out - name: Checksums run: cd out && sha256sum -- * > SHA256SUMS && cat SHA256SUMS - name: Release diff --git a/Makefile b/Makefile index 20848b8..cdafbfc 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,23 @@ maker-bin: GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o dist/squirrel-maker-darwin-amd64 ./cmd/squirrel-maker lipo -create -output dist/squirrel-maker dist/squirrel-maker-darwin-arm64 dist/squirrel-maker-darwin-amd64 +# Squirrel Maker for Windows (64-bit): the app (Wails, without cgo, so it +# builds anywhere) and the command line version, in one zip. The app asks +# for administrator rights when it starts; writing a card needs them. +WIN := dist/windows/Squirrel Maker + +.PHONY: app-windows app-windows-bundle +app-windows: payload + $(MAKE) app-windows-bundle + +app-windows-bundle: + rm -rf "$(WIN)" && mkdir -p "$(WIN)" + GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS) -H windowsgui" -o "$(WIN)/Squirrel Maker.exe" ./cmd/squirrel-app + GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o "$(WIN)/squirrel-maker.exe" ./cmd/squirrel-maker + cp LICENSE THIRD_PARTY_NOTICES.md "$(WIN)/" + rm -f "dist/Squirrel-Maker-$(VERSION)-windows.zip" + cd dist/windows && zip -qr "../Squirrel-Maker-$(VERSION)-windows.zip" "Squirrel Maker" + .PHONY: clean clean: rm -rf dist $(PAYLOAD)/files $(PAYLOAD)/files-armv6 $(PAYLOAD)/armbian diff --git a/README.md b/README.md index baa0725..499b345 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Squirrel is a replacement for [teslausb](https://github.com/marcone/teslausb), w A Raspberry Pi 3 Model B or B+ cannot act as a USB drive and does not work. - **A good SD card** of 32 GB or more: A2 or V30, or one sold for dashcams ("High Endurance"). The car writes all the time. - **A USB cable** from the car to the board's data port (see [Connecting to the car](#3-connect-it-to-the-car)). -- **A Mac** (macOS 13 or later) to make the card with Squirrel Maker. +- **A Mac** (macOS 13 or later) **or a Windows PC** (Windows 10 or 11, 64-bit) to make the card with Squirrel Maker, with a card reader. - Optional: **a NAS** with an SMB share (Synology, QNAP, a Windows share, Samba) for the clips to go to at home. Tesla puts speed, pedals, steering and position into every frame from firmware 2025.44.25 on, on HW3 and later. Older cars still record normally; the player then shows video only. @@ -39,11 +39,12 @@ Tesla puts speed, pedals, steering and position into every frame from firmware 2 ### 1. Get Squirrel Maker -Squirrel Maker is the Mac app that makes the SD card. Download `Squirrel-Maker-.zip` from the [releases](https://github.com/marcusaram/squirrel/releases), or build it yourself (see [Development](#development)). +Squirrel Maker is the app that makes the SD card, for macOS and Windows. Download it from the [releases](https://github.com/marcusaram/squirrel/releases), or build it yourself (see [Development](#development)): -The app is not signed with an Apple developer certificate. The first time, macOS says it cannot check the developer: open **System Settings → Privacy & Security** and choose **Open Anyway** below the message, or right-click the app and choose **Open**. +- **macOS:** `Squirrel-Maker-.zip`, with the app inside. It is not signed with an Apple developer certificate: the first time, macOS says it cannot check the developer. Open **System Settings → Privacy & Security** and choose **Open Anyway** below the message, or right-click the app and choose **Open**. +- **Windows:** `Squirrel-Maker--windows.zip`. Unpack it and start `Squirrel Maker.exe`. It asks for administrator rights, which writing a card needs. It is not signed either: if Windows says it protected your PC, choose **More info** and then **Run anyway**. The app uses Microsoft Edge WebView2, which Windows 10 and 11 normally have. -Squirrel Maker also runs from the command line (`squirrel-maker`), with the same steps in your browser. +Squirrel Maker also runs from the command line (`squirrel-maker`, on Windows `squirrel-maker.exe` from a terminal opened as administrator), with the same steps in your browser. ### 2. Make the card @@ -54,7 +55,7 @@ Put the SD card in your Mac, open Squirrel Maker and follow the seven steps: 3. **Log in**: the Pi's name on the network (`squirrel` by default, so `http://squirrel.local/`), a user, and a password and/or your SSH key. 4. **NAS and storage**: the NAS (optional, can be set later), how much of the card the car gets, and on a Pi 4, Pi 5 or ROCK 4C+ whether the clips go on a USB drive instead. 5. **Build image**: downloads the operating system once (checked against its checksum), and puts Squirrel and your settings in it. By default it uses the release tested with Squirrel; newer ones are offered, marked as untested. -6. **SD card**: pick the card. Only external disks and card readers are listed; your Mac asks for your password, and everything written is read back and checked. +6. **SD card**: pick the card. Only external disks and card readers are listed, never the disk your computer runs from; your Mac asks for your password (Windows asked when the app started), and everything written is read back and checked. 7. **Done**. The app remembers your answers for the next card, except passwords. @@ -227,6 +228,7 @@ make build # squirreld for arm64, armv7 and armv6 make release # squirreld archives with the install scripts make app # dist/Squirrel Maker.app and its zip (Apple silicon and Intel) make maker # dist/squirrel-maker, the command line version +make app-windows # dist/Squirrel-Maker--windows.zip: app and command line for Windows ``` `make app` and `make maker` build in two steps, which CI runs on different machines: `make payload` (squirreld, and the packages a new card installs at its first start, fetched with Docker so that the first start needs no internet), then `make app-bundle` or `make maker-bin` around it. @@ -277,6 +279,8 @@ Used in a car with a Raspberry Pi Zero 2 W: recording, keeping drives and events Tested on a Pi Zero W: making the card, the first start and the web interface; not yet in a car. +Squirrel Maker for Windows is tested on GitHub's Windows machines with a virtual disk as the card (listing, writing, inspecting and updating a Raspberry Pi card), not yet with a real card reader. How it changes a card's file systems (FAT32 for the Raspberry Pi, ext4 for the ROCK 4C+) is the same code on macOS and Windows, tested against fsck.fat, e2fsck and the Linux kernel. + Tested in containers and with test images only, not yet on the hardware: - the ROCK 4C+ (the maker's changes to the real Armbian image, the first start, a full start with a read-only system); diff --git a/cmd/squirrel-app/elevate_darwin.go b/cmd/squirrel-app/elevate_darwin.go new file mode 100644 index 0000000..671d026 --- /dev/null +++ b/cmd/squirrel-app/elevate_darwin.go @@ -0,0 +1,5 @@ +package main + +// elevate does nothing on macOS: it asks for permission itself, when a +// card is opened. +func elevate() {} diff --git a/cmd/squirrel-app/elevate_windows.go b/cmd/squirrel-app/elevate_windows.go new file mode 100644 index 0000000..5b7b7d6 --- /dev/null +++ b/cmd/squirrel-app/elevate_windows.go @@ -0,0 +1,28 @@ +package main + +import ( + "os" + "strings" + "syscall" + + "golang.org/x/sys/windows" +) + +// elevate starts the app again as administrator, through Windows's own +// prompt, when it runs without: writing a card needs it. Declined, the +// app carries on and says so when a card is to be written. +func elevate() { + if windows.GetCurrentProcessToken().IsElevated() { + return + } + exe, err := os.Executable() + if err != nil { + return + } + verb, _ := syscall.UTF16PtrFromString("runas") + file, _ := syscall.UTF16PtrFromString(exe) + args, _ := syscall.UTF16PtrFromString(strings.Join(os.Args[1:], " ")) + if err := windows.ShellExecute(0, verb, file, args, nil, windows.SW_NORMAL); err == nil { + os.Exit(0) + } +} diff --git a/cmd/squirrel-app/main.go b/cmd/squirrel-app/main.go index 60f7505..604264b 100644 --- a/cmd/squirrel-app/main.go +++ b/cmd/squirrel-app/main.go @@ -1,8 +1,10 @@ -//go:build darwin +//go:build darwin || windows -// Command squirrel-app is Squirrel Maker as a Mac app: its own window, -// Dock icon and menu, built with Wails. The page and everything behind it -// are the same as the browser version's (internal/makerui). +// Command squirrel-app is Squirrel Maker as an app for macOS and Windows: +// its own window, icon and menu, built with Wails. The page and everything +// behind it are the same as the browser version's (internal/makerui). On +// Windows it asks for administrator rights when it starts: writing a card +// needs them. package main import ( @@ -20,6 +22,7 @@ var version = "dev" var icon []byte func main() { + elevate() var app *application.App srv, err := makerui.New(makerui.Options{ Version: version, diff --git a/cmd/squirrel-maker/admin_other.go b/cmd/squirrel-maker/admin_other.go new file mode 100644 index 0000000..a6e8a93 --- /dev/null +++ b/cmd/squirrel-maker/admin_other.go @@ -0,0 +1,6 @@ +//go:build !windows + +package main + +// adminWarning: other systems ask for permission when a card is opened. +func adminWarning() string { return "" } diff --git a/cmd/squirrel-maker/admin_windows.go b/cmd/squirrel-maker/admin_windows.go new file mode 100644 index 0000000..7fe8af9 --- /dev/null +++ b/cmd/squirrel-maker/admin_windows.go @@ -0,0 +1,12 @@ +package main + +import "golang.org/x/sys/windows" + +// adminWarning says when writing a card will not work: on Windows that +// needs administrator rights. +func adminWarning() string { + if windows.GetCurrentProcessToken().IsElevated() { + return "" + } + return "Writing a card needs administrator rights: run squirrel-maker from a terminal opened as administrator." +} diff --git a/cmd/squirrel-maker/main.go b/cmd/squirrel-maker/main.go index be4bb90..95389ab 100644 --- a/cmd/squirrel-maker/main.go +++ b/cmd/squirrel-maker/main.go @@ -1,6 +1,8 @@ // Command squirrel-maker is Squirrel Maker on the command line: it serves // the maker's page on localhost and opens it in the browser, until Ctrl-C. -// The Mac app (cmd/squirrel-app) is the same maker in its own window. +// The app (cmd/squirrel-app) is the same maker in its own window. On +// Windows, run it from a terminal started as administrator: writing a +// card needs that. package main import ( @@ -13,12 +15,25 @@ import ( "net/http" "os" "os/exec" + "runtime" "github.com/marcusaram/squirrel/internal/makerui" ) var version = "dev" +// openBrowser shows url in the default browser. +func openBrowser(url string) { + switch runtime.GOOS { + case "windows": + exec.Command("rundll32", "url.dll,FileProtocolHandler", url).Start() + case "darwin": + exec.Command("open", url).Start() + default: + exec.Command("xdg-open", url).Start() + } +} + func main() { noBrowser := flag.Bool("no-browser", false, "do not open the page in the browser") addr := flag.String("listen", "127.0.0.1:0", "address for the local page") @@ -39,8 +54,11 @@ func main() { } url := fmt.Sprintf("http://%s/?t=%s", ln.Addr(), token) fmt.Printf("squirrel-maker %s\nOpen %s\nStop with Ctrl-C.\n", version, url) + if warning := adminWarning(); warning != "" { + fmt.Println(warning) + } if !*noBrowser { - exec.Command("open", url).Start() + openBrowser(url) } log.Fatal(http.Serve(ln, srv.Handler())) } diff --git a/internal/maker/disks_other.go b/internal/maker/disks_other.go index c7ba4b5..66a2bed 100644 --- a/internal/maker/disks_other.go +++ b/internal/maker/disks_other.go @@ -1,4 +1,4 @@ -//go:build !darwin +//go:build !darwin && !windows package maker diff --git a/internal/maker/disks_windows.go b/internal/maker/disks_windows.go new file mode 100644 index 0000000..05e4976 --- /dev/null +++ b/internal/maker/disks_windows.go @@ -0,0 +1,220 @@ +package maker + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/windows" +) + +type Disk struct { + ID string `json:"id"` // PhysicalDrive3 + Name string `json:"name"` + Size int64 `json:"size"` + Protocol string `json:"protocol"` +} + +// AllowDiskImages makes attached virtual disks (VHDs) count as cards. +// Only for tests. +var AllowDiskImages = os.Getenv("SQUIRREL_MAKER_ALLOW_DISK_IMAGES") == "1" + +var diskIDRe = regexp.MustCompile(`^PhysicalDrive([0-9]+)$`) + +var errNotAdmin = errors.New("writing to a card needs administrator rights; start Squirrel Maker as administrator") + +// elevated tells whether the maker runs as administrator, which raw disk +// access needs on Windows. +func elevated() bool { return windows.GetCurrentProcessToken().IsElevated() } + +// powershell runs a script without showing a console window. +func powershell(ctx context.Context, script string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", + "-Command", "$ErrorActionPreference = 'Stop'; [Console]::OutputEncoding = [Text.Encoding]::UTF8; "+script) + cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: windows.CREATE_NO_WINDOW} + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return out, fmt.Errorf("powershell: %v: %s", err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +type winDisk struct { + Number int + Name string + Size int64 + Bus string + System bool + Boot bool + ReadOnly bool +} + +func listWinDisks(ctx context.Context) ([]winDisk, error) { + out, err := powershell(ctx, `ConvertTo-Json -Compress -InputObject @(Get-Disk | ForEach-Object { + [pscustomobject]@{ Number = [int]$_.Number; Name = [string]$_.FriendlyName; Size = [int64]$_.Size; + Bus = [string]$_.BusType; System = [bool]$_.IsSystem; Boot = [bool]$_.IsBoot; ReadOnly = [bool]$_.IsReadOnly } })`) + if err != nil { + return nil, err + } + var disks []winDisk + if err := json.Unmarshal(bytes.TrimSpace(out), &disks); err != nil { + return nil, fmt.Errorf("listing disks: %w", err) + } + return disks, nil +} + +// winPart is a partition as Windows sees it. +type winPart struct { + Number int + Type int // MBR type: 12 for FAT32, 131 for Linux + Offset int64 + Size int64 + Letter string + Label string +} + +func partitions(ctx context.Context, disk int) ([]winPart, error) { + out, err := powershell(ctx, fmt.Sprintf(`ConvertTo-Json -Compress -InputObject @(Get-Partition -DiskNumber %d -ErrorAction SilentlyContinue | ForEach-Object { + $v = $_ | Get-Volume -ErrorAction SilentlyContinue + $l = if ($_.DriveLetter -and [int][char]$_.DriveLetter -ne 0) { [string]$_.DriveLetter } else { '' } + [pscustomobject]@{ Number = [int]$_.PartitionNumber; Type = [int]$_.MbrType; Offset = [int64]$_.Offset; + Size = [int64]$_.Size; Letter = $l; Label = [string]$v.FileSystemLabel } })`, disk)) + if err != nil { + return nil, err + } + var parts []winPart + if err := json.Unmarshal(bytes.TrimSpace(out), &parts); err != nil { + return nil, fmt.Errorf("listing partitions: %w", err) + } + return parts, nil +} + +func diskNumber(id string) (int, error) { + m := diskIDRe.FindStringSubmatch(id) + if m == nil { + return 0, fmt.Errorf("invalid disk %q", id) + } + return strconv.Atoi(m[1]) +} + +// checkDisk returns the disk if it is safe to overwrite: a card reader or +// USB disk, not the one Windows runs from, writable, the size of an SD +// card. +func checkDisk(ctx context.Context, id string) (Disk, error) { + n, err := diskNumber(id) + if err != nil { + return Disk{}, err + } + disks, err := listWinDisks(ctx) + if err != nil { + return Disk{}, err + } + for _, d := range disks { + if d.Number != n { + continue + } + if err := diskOK(d); err != nil { + return Disk{}, fmt.Errorf("%s %w", id, err) + } + name := d.Name + if parts, err := partitions(ctx, n); err == nil { + for _, p := range parts { + if p.Label != "" { + name = p.Label + ", " + name + break + } + } + } + return Disk{ID: id, Name: name, Size: d.Size, Protocol: d.Bus}, nil + } + return Disk{}, fmt.Errorf("%s not found", id) +} + +func diskOK(d winDisk) error { + bus := d.Bus == "USB" || d.Bus == "SD" || d.Bus == "MMC" || + AllowDiskImages && (d.Bus == "File Backed Virtual" || d.Bus == "FileBackedVirtual") + switch { + case !bus: + return fmt.Errorf("is not a card or USB disk (%s)", d.Bus) + case d.System || d.Boot: + return errors.New("holds Windows") + case d.ReadOnly: + return errors.New("is write-protected (check the lock switch on the card)") + case d.Size < 8<<30 || d.Size > 2<<40: + return fmt.Errorf("is %d GB; expected an SD card of 8 GB to 2 TB", d.Size>>30) + } + return nil +} + +// ListDisks returns the cards that can be written. +func ListDisks(ctx context.Context) ([]Disk, error) { + disks, err := listWinDisks(ctx) + if err != nil { + return nil, err + } + var out []Disk + for _, d := range disks { + if diskOK(d) == nil { + if disk, err := checkDisk(ctx, fmt.Sprintf("PhysicalDrive%d", d.Number)); err == nil { + out = append(out, disk) + } + } + } + return out, nil +} + +func openDisk(n int, write bool) (*os.File, error) { + flag := os.O_RDONLY + if write { + flag = os.O_RDWR + } + return os.OpenFile(fmt.Sprintf(`\\.\PhysicalDrive%d`, n), flag, 0) +} + +// WriteCard writes image to the card. Windows lets go of a disk's volumes +// only when they are gone, so the card is cleared first; the partition +// table is written last (see WriteRaw), so nothing is mounted halfway. +func WriteCard(ctx context.Context, image, id string, report Progress) error { + if !elevated() { + return errNotAdmin + } + d, err := checkDisk(ctx, id) + if err != nil { + return err + } + n, _ := diskNumber(id) + fi, err := os.Stat(image) + if err != nil { + return err + } + if fi.Size() > d.Size { + return fmt.Errorf("the image (%d GB) does not fit on %s", fi.Size()>>30, d.Name) + } + report("Preparing the SD card", -1) + if _, err := powershell(ctx, fmt.Sprintf(`$d = Get-Disk -Number %d + if ($d.PartitionStyle -ne 'RAW') { Clear-Disk -Number %d -RemoveData -RemoveOEM -Confirm:$false }`, n, n)); err != nil { + return err + } + dev, err := openDisk(n, true) + if err != nil { + return err + } + err = WriteRaw(image, dev, report) + dev.Close() + if err != nil { + return err + } + report("Finishing", -1) + _, err = powershell(ctx, fmt.Sprintf(`Update-Disk -Number %d`, n)) + return err +} diff --git a/internal/maker/disks_windows_test.go b/internal/maker/disks_windows_test.go new file mode 100644 index 0000000..58d26e3 --- /dev/null +++ b/internal/maker/disks_windows_test.go @@ -0,0 +1,110 @@ +package maker + +import ( + "context" + "fmt" + "math/rand" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "testing/fstest" +) + +// A virtual disk as the card: listed, written, made a Raspberry Pi card, +// inspected and updated, on Windows itself. Opt-in, as administrator +// (GitHub's Windows runners are): SQUIRREL_TEST_VHD=C:\path\card.vhdx. +func TestWindowsCard(t *testing.T) { + vhd := os.Getenv("SQUIRREL_TEST_VHD") + if vhd == "" { + t.Skip("set SQUIRREL_TEST_VHD to a path for a virtual disk") + } + if !elevated() { + t.Skip("needs administrator rights") + } + AllowDiskImages = true + defer func() { AllowDiskImages = false }() + ctx := context.Background() + + diskpart := func(script string) { + t.Helper() + f := filepath.Join(t.TempDir(), "script.txt") + os.WriteFile(f, []byte(script), 0o644) + if out, err := exec.Command("diskpart", "/s", f).CombinedOutput(); err != nil { + t.Fatalf("diskpart: %v\n%s", err, out) + } + } + os.Remove(vhd) + diskpart(fmt.Sprintf("create vdisk file=\"%s\" maximum=8400 type=expandable\nattach vdisk\n", vhd)) + defer diskpart(fmt.Sprintf("select vdisk file=\"%s\"\ndetach vdisk\n", vhd)) + out, err := powershell(ctx, fmt.Sprintf(`(Get-DiskImage -ImagePath '%s' | Get-Disk).Number`, vhd)) + if err != nil { + t.Fatal(err) + } + n, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil { + t.Fatalf("disk number %q", out) + } + id := fmt.Sprintf("PhysicalDrive%d", n) + + disks, err := ListDisks(ctx) + if err != nil { + t.Fatal(err) + } + found := false + for _, d := range disks { + found = found || d.ID == id + } + if !found { + t.Fatalf("%s not among the cards: %+v", id, disks) + } + + // Writing an image: WriteCard reads it back and compares. + img := filepath.Join(t.TempDir(), "card.img") + data := make([]byte, 64<<20) + rand.New(rand.NewSource(1)).Read(data) + os.WriteFile(img, data, 0o644) + if err := WriteCard(ctx, img, id, func(string, float64) {}); err != nil { + t.Fatal(err) + } + + // A Raspberry Pi card from before: a FAT32 boot partition with + // Squirrel's folder, started once. + if _, err := powershell(ctx, fmt.Sprintf(`$d = Get-Disk -Number %[1]d + if ($d.PartitionStyle -ne 'RAW') { Clear-Disk -Number %[1]d -RemoveData -RemoveOEM -Confirm:$false } + Initialize-Disk -Number %[1]d -PartitionStyle MBR + New-Partition -DiskNumber %[1]d -Size 512MB -MbrType FAT32 -AssignDriveLetter | + Format-Volume -FileSystem FAT32 -NewFileSystemLabel bootfs -Confirm:$false | Out-Null`, n)); err != nil { + t.Fatal(err) + } + parts, err := partitions(ctx, n) + if err != nil || len(parts) == 0 || parts[0].Letter == "" { + t.Fatalf("partitions: %+v %v", parts, err) + } + sq := parts[0].Letter + `:\squirrel` + os.MkdirAll(sq, 0o755) + arm64 := []byte{0x7f, 'E', 'L', 'F', 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0xB7, 0} + os.WriteFile(filepath.Join(sq, "squirreld"), arm64, 0o644) + os.WriteFile(filepath.Join(sq, "firstboot.sh"), []byte("old"), 0o644) + os.WriteFile(filepath.Join(sq, "version"), []byte("0.1.0\n"), 0o644) + os.WriteFile(filepath.Join(sq, "done"), nil, 0o644) + + info, err := InspectCard(ctx, id) + if err != nil || info.Board != "rpi" || info.Version != "0.1.0" || info.Pending { + t.Fatalf("inspect: %+v %v", info, err) + } + payload := fstest.MapFS{ + "files/squirreld": {Data: append(arm64, []byte("new")...)}, + "files/firstboot.sh": {Data: []byte("new")}, + "files/version": {Data: []byte("9.9.9\n")}, + } + info, err = UpdateCard(ctx, id, payload, func(string, float64) {}) + if err != nil || info.Version != "9.9.9" || !info.Pending { + t.Fatalf("update: %+v %v", info, err) + } + if got, _ := os.ReadFile(filepath.Join(sq, "firstboot.sh")); string(got) != "new" { + t.Errorf("firstboot.sh: %q", got) + } +} diff --git a/internal/maker/update_other.go b/internal/maker/update_other.go index 2666b06..824a138 100644 --- a/internal/maker/update_other.go +++ b/internal/maker/update_other.go @@ -1,4 +1,4 @@ -//go:build !darwin +//go:build !darwin && !windows package maker diff --git a/internal/maker/update_windows.go b/internal/maker/update_windows.go new file mode 100644 index 0000000..06c4950 --- /dev/null +++ b/internal/maker/update_windows.go @@ -0,0 +1,130 @@ +package maker + +import ( + "context" + "errors" + "fmt" + "io/fs" +) + +// isArmbian: one Linux partition at 16 MiB and nothing else, as Armbian +// writes it. +func isArmbian(parts []winPart) bool { + return len(parts) == 1 && parts[0].Type == 0x83 && parts[0].Offset == armbianRootStart +} + +// bootDir returns the folder of a Raspberry Pi card's FAT boot partition, +// giving it a drive letter when Windows has not. +func bootDir(ctx context.Context, disk int, parts []winPart) (string, error) { + if len(parts) == 0 || (parts[0].Type != 0x0C && parts[0].Type != 0x0B && parts[0].Type != 0x0E && parts[0].Type != 0x06) { + return "", errors.New("this card has no Squirrel") + } + letter := parts[0].Letter + if letter == "" { + if _, err := powershell(ctx, fmt.Sprintf(`Add-PartitionAccessPath -DiskNumber %d -PartitionNumber %d -AssignDriveLetter`, + disk, parts[0].Number)); err != nil { + return "", err + } + again, err := partitions(ctx, disk) + if err != nil || len(again) == 0 || again[0].Letter == "" { + return "", errors.New("the card's boot partition got no drive letter") + } + letter = again[0].Letter + } + return letter + `:\`, nil +} + +// InspectCard tells whether a card holds Squirrel, for which board and +// which version. +func InspectCard(ctx context.Context, id string) (CardInfo, error) { + d, err := checkDisk(ctx, id) + if err != nil { + return CardInfo{}, err + } + n, _ := diskNumber(id) + parts, err := partitions(ctx, n) + if err != nil { + return CardInfo{}, err + } + if isArmbian(parts) { + if !elevated() { + return CardInfo{Disk: d, Board: "rock4cplus"}, nil + } + dev, err := openDisk(n, false) + if err != nil { + return CardInfo{}, err + } + defer dev.Close() + r, err := openRoot(dev) + if err != nil { + return CardInfo{}, err + } + info, err := inspectArmbian(r) + info.Disk = d + return info, err + } + dir, err := bootDir(ctx, n, parts) + if err != nil { + return CardInfo{}, err + } + info, err := inspectPiBoot(dir) + info.Disk = d + return info, err +} + +// UpdateCard puts the Squirrel of this app on a card that already holds +// it. all is the whole payload. +func UpdateCard(ctx context.Context, id string, all fs.FS, report Progress) (CardInfo, error) { + if !elevated() { + return CardInfo{}, errNotAdmin + } + d, err := checkDisk(ctx, id) + if err != nil { + return CardInfo{}, err + } + n, _ := diskNumber(id) + parts, err := partitions(ctx, n) + if err != nil { + return CardInfo{}, err + } + if isArmbian(parts) { + dev, err := openDisk(n, true) + if err != nil { + return CardInfo{}, err + } + defer dev.Close() + r, err := openRoot(dev) + if err != nil { + return CardInfo{}, err + } + report("Updating the card", -1) + if err := updateArmbian(r, all, report); err != nil { + return CardInfo{}, err + } + if err := dev.Sync(); err != nil { + return CardInfo{}, err + } + info, err := inspectArmbian(r) + info.Disk = d + return info, err + } + dir, err := bootDir(ctx, n, parts) + if err != nil { + return CardInfo{}, err + } + if _, err := inspectPiBoot(dir); err != nil { + return CardInfo{}, err + } + report("Updating the card", -1) + if err := updatePiBoot(dir, all); err != nil { + return CardInfo{}, err + } + info, err := inspectPiBoot(dir) + info.Disk = d + if err != nil { + return info, err + } + // Written through to the card before it is pulled out. + _, err = powershell(ctx, fmt.Sprintf(`Write-VolumeCache -DriveLetter %s`, dir[:1])) + return info, err +} diff --git a/internal/maker/write.go b/internal/maker/write.go index c0e396c..e1b348c 100644 --- a/internal/maker/write.go +++ b/internal/maker/write.go @@ -12,8 +12,14 @@ import ( const chunk = 4 << 20 -// WriteRaw copies image onto an opened raw disk and reads it back to -// compare. Writes are kept in whole 512 byte sectors. +// headLast is how much of the start of the image (the partition table and +// what follows) is written last: until then the card has no partitions a +// computer would try to mount while the rest is written. +const headLast = 1 << 20 + +// WriteRaw copies image onto an opened raw disk, the first megabyte last, +// and reads it back to compare. Writes are kept in whole 512 byte sectors +// at sector offsets, which raw disks on macOS and Windows require. func WriteRaw(image string, out *os.File, report Progress) error { device := out.Name() in, err := os.Open(image) @@ -26,44 +32,54 @@ func WriteRaw(image string, out *os.File, report Progress) error { return err } size := fi.Size() - want := sha256.New() buf := make([]byte, chunk) var done int64 - for done < size { - n, err := io.ReadFull(in, buf) - if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) { - return err - } - if n == 0 { - break + write := func(from, to int64) error { + for at := from; at < to; { + n := min(int64(chunk), to-at) + m, err := in.ReadAt(buf[:n], at) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + w := (int64(m) + 511) / 512 * 512 + clear(buf[m:w]) + if _, err := out.WriteAt(buf[:w], at); err != nil { + return fmt.Errorf("writing to %s: %w", device, err) + } + at += n + done += n + report("Writing the SD card", float64(done)/float64(size)) } - want.Write(buf[:n]) - w := (n + 511) / 512 * 512 - clear(buf[n:w]) - if _, err := out.Write(buf[:w]); err != nil { - return fmt.Errorf("writing to %s: %w", device, err) - } - done += int64(n) - report("Writing the SD card", float64(done)/float64(size)) + return nil } - // Raw disk nodes on macOS are unbuffered and do not support fsync. - if err := out.Sync(); err != nil && !errors.Is(err, syscall.ENOTTY) { + head := min(int64(headLast), size) + if err := write(head, size); err != nil { return err } - back := out - if _, err := back.Seek(0, io.SeekStart); err != nil { + if err := write(0, head); err != nil { return err } - got := sha256.New() + // Raw disk nodes on macOS are unbuffered and do not support fsync. + if err := out.Sync(); err != nil && !errors.Is(err, syscall.ENOTTY) { + return err + } + + want, got := sha256.New(), sha256.New() + cmp := make([]byte, chunk) var read int64 for read < size { - w := min(int64(chunk), (size-read+511)/512*512) - n, err := io.ReadFull(back, buf[:w]) - if err != nil && n == 0 { + n := min(int64(chunk), size-read) + w := (n + 511) / 512 * 512 + if _, err := in.ReadAt(buf[:n], read); err != nil && !errors.Is(err, io.EOF) { + return err + } + m, err := out.ReadAt(cmp[:w], read) + if int64(m) < n { return fmt.Errorf("reading back: %w", err) } - got.Write(buf[:min(int64(n), size-read)]) - read += int64(n) + want.Write(buf[:n]) + got.Write(cmp[:n]) + read += n report("Verifying", float64(read)/float64(size)) } if !bytes.Equal(got.Sum(nil), want.Sum(nil)) { diff --git a/internal/maker/write_test.go b/internal/maker/write_test.go new file mode 100644 index 0000000..e230452 --- /dev/null +++ b/internal/maker/write_test.go @@ -0,0 +1,45 @@ +package maker + +import ( + "bytes" + "math/rand" + "os" + "path/filepath" + "testing" +) + +// WriteRaw puts the whole image on the disk, the partition table last, +// also when the image does not end on a sector boundary. +func TestWriteRaw(t *testing.T) { + dir := t.TempDir() + data := make([]byte, 9<<20+123) + rand.New(rand.NewSource(1)).Read(data) + img := filepath.Join(dir, "card.img") + os.WriteFile(img, data, 0o644) + dev, err := os.OpenFile(filepath.Join(dir, "disk"), os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + t.Fatal(err) + } + defer dev.Close() + var steps []string + var last float64 + err = WriteRaw(img, dev, func(s string, f float64) { + if len(steps) == 0 || steps[len(steps)-1] != s { + steps = append(steps, s) + } + last = f + }) + if err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(dev.Name()) + if len(got) < len(data) || !bytes.Equal(got[:len(data)], data) { + t.Fatalf("disk holds %d bytes, not the image", len(got)) + } + if len(got)%512 != 0 || got[len(data)] != 0 { + t.Errorf("the last sector is not padded with zeros (%d bytes)", len(got)) + } + if len(steps) != 2 || steps[0] != "Writing the SD card" || steps[1] != "Verifying" || last != 1 { + t.Errorf("progress: %v, last %v", steps, last) + } +} diff --git a/internal/makerui/country_darwin.go b/internal/makerui/country_darwin.go new file mode 100644 index 0000000..5156e37 --- /dev/null +++ b/internal/makerui/country_darwin.go @@ -0,0 +1,18 @@ +package makerui + +import ( + "os/exec" + "strings" +) + +// systemCountry is the region the Mac is set to, e.g. NL from nl_NL. +func systemCountry() string { + out, err := exec.Command("defaults", "read", "-g", "AppleLocale").Output() + if err != nil { + return "" + } + if _, c, ok := strings.Cut(strings.TrimSpace(string(out)), "_"); ok && len(c) >= 2 { + return strings.ToUpper(c[:2]) + } + return "" +} diff --git a/internal/makerui/country_other.go b/internal/makerui/country_other.go new file mode 100644 index 0000000..c5b47cf --- /dev/null +++ b/internal/makerui/country_other.go @@ -0,0 +1,6 @@ +//go:build !darwin && !windows + +package makerui + +// systemCountry: on other systems the time zone tells the country. +func systemCountry() string { return "" } diff --git a/internal/makerui/country_windows.go b/internal/makerui/country_windows.go new file mode 100644 index 0000000..2586439 --- /dev/null +++ b/internal/makerui/country_windows.go @@ -0,0 +1,22 @@ +package makerui + +import ( + "os/exec" + "regexp" + "strings" + "syscall" + + "golang.org/x/sys/windows" +) + +// systemCountry is the country or region Windows is set to, e.g. NL. +func systemCountry() string { + cmd := exec.Command("powershell.exe", "-NoProfile", "-NonInteractive", "-Command", + "[System.Globalization.RegionInfo]::CurrentRegion.TwoLetterISORegionName") + cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: windows.CREATE_NO_WINDOW} + out, err := cmd.Output() + if c := strings.ToUpper(strings.TrimSpace(string(out))); err == nil && regexp.MustCompile(`^[A-Z]{2}$`).MatchString(c) { + return c + } + return "" +} diff --git a/internal/makerui/makerui.go b/internal/makerui/makerui.go index c300905..b1befc1 100644 --- a/internal/makerui/makerui.go +++ b/internal/makerui/makerui.go @@ -17,6 +17,7 @@ import ( "os/user" "path/filepath" "regexp" + "runtime" "slices" "strings" "sync" @@ -204,10 +205,8 @@ func (a *Server) defaults(w http.ResponseWriter, r *http.Request) { // better than the language the Mac is set to. if c := countryForTimezone(s.Timezone); c != "" { s.Country = c - } else if out, err := exec.Command("defaults", "read", "-g", "AppleLocale").Output(); err == nil { - if _, c, ok := strings.Cut(strings.TrimSpace(string(out)), "_"); ok && len(c) >= 2 { - s.Country = strings.ToUpper(c[:2]) - } + } else if c := systemCountry(); c != "" { + s.Country = c } if home, err := os.UserHomeDir(); err == nil { for _, k := range []string{"id_ed25519.pub", "id_ecdsa.pub", "id_rsa.pub"} { @@ -229,6 +228,7 @@ func (a *Server) defaults(w http.ResponseWriter, r *http.Request) { "payload_ok": payloadErr == nil, "version": a.opts.Version, "native": a.opts.Native != nil, + "platform": runtime.GOOS, }) } @@ -480,8 +480,12 @@ func (a *Server) reveal(w http.ResponseWriter, r *http.Request) { image := a.image a.mu.Unlock() if image != "" { - // Finder shows the image selected, as the app's own window would. - exec.Command("open", "-R", image).Run() + // Finder or Explorer shows the image selected. + if runtime.GOOS == "windows" { + exec.Command("explorer", "/select,"+image).Run() + } else { + exec.Command("open", "-R", image).Run() + } } writeJSON(w, map[string]bool{"ok": true}) } diff --git a/internal/makerui/static/i18n.js b/internal/makerui/static/i18n.js index ea0a264..fdb7f33 100644 --- a/internal/makerui/static/i18n.js +++ b/internal/makerui/static/i18n.js @@ -84,7 +84,7 @@ const NL = { 'Hide': 'Verberg', 'May be empty if you use an SSH key.': 'Mag leeg als je een SSH-sleutel gebruikt.', 'SSH key': 'SSH-sleutel', - "This Mac's public key is already filled in.": 'De publieke sleutel van deze Mac staat al ingevuld.', + "This computer's public key is already filled in.": 'De publieke sleutel van deze computer staat al ingevuld.', 'Time zone': 'Tijdzone', 'NAS and storage': 'NAS en opslag', 'Once the car is home, the Pi uploads Sentry and saved clips to a shared folder on your NAS.': @@ -100,8 +100,8 @@ const NL = { 'Build the image': 'Image maken', 'SD card': 'SD-kaart', 'Put it on the SD card': 'Op de SD-kaart zetten', - 'Insert the card in your Mac. Everything on it is erased. The car writes all the time, so the card matters most: take an A2 or V30 card of 32 GB or more, or one sold for dashcams (High Endurance).': - 'Steek de kaart in je Mac. Alles wat erop staat wordt gewist. De auto schrijft de hele tijd, dus de kaart doet er het meest toe: neem een A2- of V30-kaart van 32 GB of meer, of een kaart voor dashcams (High Endurance).', + 'Insert the card in your computer. Everything on it is erased. The car writes all the time, so the card matters most: take an A2 or V30 card of 32 GB or more, or one sold for dashcams (High Endurance).': + 'Steek de kaart in je computer. Alles wat erop staat wordt gewist. De auto schrijft de hele tijd, dus de kaart doet er het meest toe: neem een A2- of V30-kaart van 32 GB of meer, of een kaart voor dashcams (High Endurance).', 'Run the SD card faster (Zero W, Zero 2 W, Pi 3)': 'De SD-kaart sneller laten werken (Zero W, Zero 2 W, Pi 3)', 'These boards drive the card at 50 MHz, about 20 MB/s: less than the car sends. This raises it to up to 100 MHz. That is beyond the SD card standard: a good A2 or V30 card usually copes, but a card that does not gives read and write errors and can lose recordings. Leave it off unless you want to try; Pi 4 and 5 do not need it.': 'Deze boards sturen de kaart op 50 MHz aan, ongeveer 20 MB/s: minder dan de auto aanlevert. Dit verhoogt dat tot hooguit 100 MHz. Dat gaat buiten de SD-standaard: een goede A2- of V30-kaart kan het meestal aan, maar een kaart die het niet aankan geeft lees- en schrijffouten en kan opnames kwijtraken. Laat het uit tenzij je het wilt proberen; Pi 4 en 5 hebben het niet nodig.', @@ -119,8 +119,8 @@ const NL = { 'Go to': 'Ga naar', ', or join the wifi Squirrel if the Pi does not find your network.': ', of verbind met de wifi Squirrel als de Pi je netwerk niet vindt.', - 'Not working? Put the card in your Mac: squirrel/firstboot.log tells where it stopped.': - 'Lukt het niet? Steek de kaart in je Mac: squirrel/firstboot.log vertelt waar het stopte.', + 'Not working? Put the card in your computer: squirrel/firstboot.log tells where it stopped.': + 'Lukt het niet? Steek de kaart in je computer: squirrel/firstboot.log vertelt waar het stopte.', 'Make another card': 'Nog een kaart maken', 'Back': 'Vorige', 'Next': 'Volgende', @@ -131,6 +131,7 @@ const NL = { 'enter the password again': 'wachtwoord opnieuw invullen', 'Image ready.': 'Image klaar.', 'Show in Finder': 'Toon in Finder', + 'Show in Explorer': 'Toon in Verkenner', 'Not started': 'Niet gestart', 'No SD card found. Insert one and choose Search again.': 'Geen SD-kaart gevonden. Steek er een in en kies Opnieuw zoeken.', 'Card': 'Kaart', @@ -182,6 +183,7 @@ const NL = { 'Writing the SD card': 'SD-kaart beschrijven', 'Verifying': 'Controleren', 'Ejecting the card': 'Kaart uitwerpen', + 'Finishing': 'Afronden', 'a task is already running': 'er loopt al een taak', 'build the image first': 'maak eerst de image', 'wait until the current task is done': 'wacht tot de huidige taak klaar is', @@ -202,6 +204,8 @@ const NL = { 'a wifi name must be 1 to 32 characters': 'een wifi-naam moet 1 tot 32 tekens hebben', 'writing cards is only supported on macOS for now': 'kaarten schrijven kan voorlopig alleen op macOS', 'adapting the image is only supported on macOS for now': 'de image aanpassen kan voorlopig alleen op macOS', + 'writing to a card needs administrator rights; start Squirrel Maker as administrator': + 'naar een kaart schrijven vraagt beheerdersrechten; start Squirrel Maker als administrator', }; // Server messages with details filled in. @@ -214,6 +218,8 @@ const NL_PATTERNS = [ [/^(\S+) is not a whole disk$/, '$1 is geen hele schijf'], [/^(\S+) is an internal disk$/, '$1 is een interne schijf'], [/^(\S+) is not a physical disk$/, '$1 is geen fysieke schijf'], + [/^(\S+) is not a card or USB disk \((.*)\)$/, '$1 is geen kaart of USB-schijf ($2)'], + [/^(\S+) holds Windows$/, 'op $1 staat Windows'], [/^(\S+) is write-protected \(check the lock switch on the card\)$/, '$1 is beveiligd tegen schrijven (schuifje op de kaart?)'], [/^(\S+) is (\d+) GB; expected an SD card of 8 GB to 2 TB$/, '$1 is $2 GB; een SD-kaart van 8 GB tot 2 TB wordt verwacht'], [/^the image \((\d+) GB\) does not fit on (.+)$/, 'de image ($1 GB) past niet op $2'], diff --git a/internal/makerui/static/index.html b/internal/makerui/static/index.html index 401dec2..836ab09 100644 --- a/internal/makerui/static/index.html +++ b/internal/makerui/static/index.html @@ -53,7 +53,7 @@

Logging in to the Pi

- + @@ -94,7 +94,7 @@

Build the image

Put it on the SD card

-

Insert the card in your Mac. Everything on it is erased. The car writes all the time, so the card matters most: take an A2 or V30 card of 32 GB or more, or one sold for dashcams (High Endurance).

+

Insert the card in your computer. Everything on it is erased. The car writes all the time, so the card matters most: take an A2 or V30 card of 32 GB or more, or one sold for dashcams (High Endurance).

@@ -110,7 +110,7 @@

The card is ready

  • Wait a moment. The first start takes a few minutes. The LED blinks fast while the Pi is busy.
  • Open it. Go to http://squirrel.local/, or join the wifi Squirrel if the Pi does not find your network.
    • -

    Not working? Put the card in your Mac: squirrel/firstboot.log tells where it stopped.

    +

    Not working? Put the card in your computer: squirrel/firstboot.log tells where it stopped.

    diff --git a/internal/makerui/static/maker.js b/internal/makerui/static/maker.js index 889f792..2c058b7 100644 --- a/internal/makerui/static/maker.js +++ b/internal/makerui/static/maker.js @@ -7,6 +7,7 @@ const form = $('#settings'); // In the Mac app the page runs in a web view without confirm() or // alert(); the app's own dialogs answer instead. let NATIVE = false; +let PLATFORM = ''; async function ask(message, ok) { if (!NATIVE) return confirm(message); @@ -106,6 +107,11 @@ function fill(s) { else if (el.type === 'radio') el.checked = el.value === (v || (a === 'board' ? 'rpi' : '')); else if (v != null) el.value = v; } + // Where the computer could not tell the time zone by its IANA name + // (Windows), the browser can. + if (!form.elements.timezone.value) { + try { form.elements.timezone.value = Intl.DateTimeFormat().resolvedOptions().timeZone || ''; } catch {} + } form.elements.usb_storage.checked = s.storage === 'usb'; form.elements.sd_overclock.checked = !!s.sd_overclock; savedImage = s.image || ''; @@ -213,7 +219,7 @@ form.addEventListener('submit', async e => { if (j && j.done && !j.error) { const p = document.createElement('p'); p.className = 'result'; - p.innerHTML = `${t('Image ready.')} `; + p.innerHTML = `${t('Image ready.')} `; $('button', p).onclick = () => api('reveal', { method: 'POST' }); box.append(p); built = true; @@ -518,6 +524,7 @@ translatePage(); const d = await api('defaults'); // The Mac app quits with its menu (Cmd-Q), not from the page. NATIVE = !!d.native; + PLATFORM = d.platform || ''; $('#quit').hidden = NATIVE; boards = d.boards || []; appVersion = d.version; From 5250487639f004d177a21fcb6fe5297e65e80ae0 Mon Sep 17 00:00:00 2001 From: Marcus Aram <4435760+marcusaram@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:34:06 +0100 Subject: [PATCH 4/8] Squirrel Maker for Windows: icon, manifest and version in the exe go-winres (pinned as a go.mod tool) makes the resources when the Windows build runs: the squirrel icon, a manifest that asks for administrator rights for the app and none for the command line, and version details. The generated .syso files are not committed. --- .gitignore | 2 ++ Makefile | 14 ++++++++++++-- go.mod | 10 ++++++++++ go.sum | 16 ++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9184f85..38a0f63 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ /.claude/ /internal/makerui/payload/armbian/ /internal/makerui/payload/files-armv6/ +# Windows resources, made by make app-windows +*.syso diff --git a/Makefile b/Makefile index cdafbfc..67c2461 100644 --- a/Makefile +++ b/Makefile @@ -147,9 +147,15 @@ maker-bin: lipo -create -output dist/squirrel-maker dist/squirrel-maker-darwin-arm64 dist/squirrel-maker-darwin-amd64 # Squirrel Maker for Windows (64-bit): the app (Wails, without cgo, so it -# builds anywhere) and the command line version, in one zip. The app asks -# for administrator rights when it starts; writing a card needs them. +# builds anywhere) and the command line version, in one zip. Each gets a +# Windows resource (go-winres, pinned in go.mod) with the squirrel icon, +# a manifest and the version; the app's manifest asks for administrator +# rights when it starts, which writing a card needs. WIN := dist/windows/Squirrel Maker +# The file version must be numbers: 0.16.0 from v0.16.0-rc1, 0.0.0 from dev. +WINVER = $(shell echo "$(VERSION)" | sed -E -e 's/^v?([0-9]+(\.[0-9]+){0,3}).*/\1/' -e t -e 's/.*/0.0.0/') +WINRES = go tool go-winres simply --arch amd64 --icon cmd/squirrel-app/icon.png --product-name "Squirrel Maker" \ + --product-version "$(VERSION)" --file-version "$(WINVER)" --copyright "Marcus Aram, MIT license" .PHONY: app-windows app-windows-bundle app-windows: payload @@ -157,6 +163,10 @@ app-windows: payload app-windows-bundle: rm -rf "$(WIN)" && mkdir -p "$(WIN)" + $(WINRES) --manifest gui --admin --file-description "Squirrel Maker" \ + --original-filename "Squirrel Maker.exe" --out cmd/squirrel-app/rsrc + $(WINRES) --manifest cli --file-description "Squirrel Maker (command line)" \ + --original-filename squirrel-maker.exe --out cmd/squirrel-maker/rsrc GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS) -H windowsgui" -o "$(WIN)/Squirrel Maker.exe" ./cmd/squirrel-app GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o "$(WIN)/squirrel-maker.exe" ./cmd/squirrel-maker cp LICENSE THIRD_PARTY_NOTICES.md "$(WIN)/" diff --git a/go.mod b/go.mod index 74ab128..4a12668 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/adrg/xdg v0.5.3 // indirect github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect github.com/coder/websocket v1.8.14 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect @@ -25,6 +26,15 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/tc-hib/go-winres v0.3.3 // indirect + github.com/tc-hib/winres v0.3.1 // indirect + github.com/urfave/cli/v2 v2.25.7 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/crypto v0.57.0 // indirect + golang.org/x/image v0.45.0 // indirect golang.org/x/net v0.60.0 // indirect ) + +tool github.com/tc-hib/go-winres diff --git a/go.sum b/go.sum index 2e73c23..2379bf6 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqL github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc/go.mod h1:uvR42Hb/t52HQd7x5/ZLzZEK8oihrFpgnodIJ1vte2E= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -44,8 +46,12 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -55,16 +61,26 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tc-hib/go-winres v0.3.3 h1:DQ50qlvDVhqrDOY0svTxZFZWfKWtZtfnXXHPCw6lqh0= +github.com/tc-hib/go-winres v0.3.3/go.mod h1:5NGzOtuvjSqnpIEi2o1h48MKZzP9olvrf+PeY2t1uoA= +github.com/tc-hib/winres v0.3.1 h1:CwRjEGrKdbi5CvZ4ID+iyVhgyfatxFoizjPhzez9Io4= +github.com/tc-hib/winres v0.3.1/go.mod h1:C/JaNhH3KBvhNKVbvdlDWkbMDO9H4fKKDaN7/07SSuk= github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/wailsapp/wails/v3 v3.0.0 h1:lBe89kFu23bVz1dc1H9v0srkHQgWsy7BAKpH4sLQuDw= github.com/wailsapp/wails/v3 v3.0.0/go.mod h1:U3p3/QfSer2XxXM3XWhwis9O7LvvuNKN/AcGoEFD+K8= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= From 42bf16513a5a8fa894f99f558e2a3a9f6fdaf5c6 Mon Sep 17 00:00:00 2001 From: Marcus Aram <4435760+marcusaram@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:04:18 +0100 Subject: [PATCH 5/8] README: the ROCK 4C+ as a board for the car too, and computer instead of Mac --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 499b345..969a28d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Squirrel is a replacement for [teslausb](https://github.com/marcone/teslausb), w - Raspberry Pi **Zero 2 W**: the usual choice, small, powered by the car through the same cable. - Raspberry Pi **4** or **5**: faster, and can keep the clips on a USB SSD. - Original Raspberry Pi **Zero W**: works, but slowly (one core, 2.4 GHz wifi only). - - Radxa **ROCK 4C+**. + - Radxa **ROCK 4C+**: fast (six cores), and can keep the clips on a USB SSD. Needs its own power next to the cable from the car. A Raspberry Pi 3 Model B or B+ cannot act as a USB drive and does not work. - **A good SD card** of 32 GB or more: A2 or V30, or one sold for dashcams ("High Endurance"). The car writes all the time. @@ -48,7 +48,7 @@ Squirrel Maker also runs from the command line (`squirrel-maker`, on Windows `sq ### 2. Make the card -Put the SD card in your Mac, open Squirrel Maker and follow the seven steps: +Put the SD card in your computer, open Squirrel Maker and follow the seven steps: 1. **Board**: which board the card is for. 2. **Wifi**: your home network (for the NAS) and perhaps your phone's hotspot, and your country. A Pi Zero W only sees 2.4 GHz networks. Also the password for the Pi's own wifi, which it starts when it finds no known network (see [Wifi in the car](#wifi-in-the-car)). @@ -141,11 +141,11 @@ The car's drive is disconnected meanwhile, nothing is deleted for space, and a r ### Updating -Put the card in your Mac, open Squirrel Maker and choose **Update it** on the first page. It shows the Squirrel version on each card and puts this version on the one you pick. The board installs it at its next start; settings, wifi networks and kept clips stay. +Put the card in your computer, open Squirrel Maker and choose **Update it** on the first page. It shows the Squirrel version on each card and puts this version on the one you pick. The board installs it at its next start; settings, wifi networks and kept clips stay. ## Troubleshooting -**The LED flickers very fast after the first start.** The first start failed and will try again at the next start. Put the card in your Mac and open `squirrel/firstboot.log` on it (on a ROCK 4C+ the log is also put on the USB drive as `squirrel-firstboot.txt`). If the card was made with an older Squirrel Maker, update or remake it with the latest. +**The LED flickers very fast after the first start.** The first start failed and will try again at the next start. Put the card in your computer and open `squirrel/firstboot.log` on it (on a ROCK 4C+ the log is also put on the USB drive as `squirrel-firstboot.txt`). If the card was made with an older Squirrel Maker, update or remake it with the latest. **The car does not see the drive.** Check that the cable goes to the board's data port (see [Connect it to the car](#3-connect-it-to-the-car)) and is a data cable, not a charging-only one. The Status page shows the USB state; "configured" means the car has the drive. After a cold start the car can take a minute to notice a new drive. From c6c10fa35b3d55a945ce01d09cacfe16bc43e6fa Mon Sep 17 00:00:00 2001 From: Marcus Aram <4435760+marcusaram@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:14:49 +0100 Subject: [PATCH 6/8] README: describe the ROCK 4C+ USB device mode instead of pointing to the teslausb wiki --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 969a28d..1909597 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ The original Zero W gets 32-bit Raspberry Pi OS with an ARMv6 build of squirreld ### Radxa ROCK 4C+ -The ROCK 4C+ gets [Armbian](https://www.armbian.com/) (Debian trixie, minimal). There is no stable Armbian release for this board, only community builds; Squirrel Maker uses one that was checked. Armbian keeps everything on ext4, which macOS and Windows cannot write, so Squirrel Maker writes its files itself, without mounting the card: it only touches a file system that is clean and uses ext4 features it knows, and reads everything back afterwards. Updating such a card asks for your password. At the end of the first start the board restarts once; the car then sees the drive. Ethernet stays with Armbian's own network setup; NetworkManager handles wifi and the hotspot. The USB device mode follows the [teslausb wiki](https://github.com/marcone/teslausb/wiki/Rock-Pi-4C-plus-Installation). +The ROCK 4C+ gets [Armbian](https://www.armbian.com/) (Debian trixie, minimal). There is no stable Armbian release for this board, only community builds; Squirrel Maker uses one that was checked. Armbian keeps everything on ext4, which macOS and Windows cannot write, so Squirrel Maker writes its files itself, without mounting the card: it only touches a file system that is clean and uses ext4 features it knows, and reads everything back afterwards. Updating such a card asks for your password on a Mac. At the end of the first start the board restarts once; the car then sees the drive. Ethernet stays with Armbian's own network setup; NetworkManager handles wifi and the hotspot. Normally both USB 3 ports are host ports. The first start adds a device tree overlay (`deploy/armbian/dwc3-0-device.dts`) that puts the RK3399's first USB 3 controller (`dwc3_0`, the upper blue port) in peripheral mode; it takes effect after the restart. From then on squirreld offers the car's drive on that port through the kernel's USB gadget, as on a Pi (where `dtoverlay=dwc2` in `config.txt` does the same for the port marked USB). ### Clips on a USB drive From 4812f7b6867db61852bdc7e99c10ddc512f51e55 Mon Sep 17 00:00:00 2001 From: Marcus Aram <4435760+marcusaram@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:18:06 +0100 Subject: [PATCH 7/8] Review: Flush's promise, Explorer selection with spaces, one disk listing - ext4: say that a failed Flush can leave part of the change written. - Show in Explorer quotes only the path, so a path with spaces works. - ListDisks names each card from the listing it has, instead of listing all disks again per card. - README: updating a ROCK 4C+ card asks for a password on a Mac only. --- README.md | 2 +- internal/ext4/ext4.go | 6 ++++-- internal/ext4/write.go | 3 ++- internal/maker/disks_windows.go | 27 +++++++++++++++------------ internal/makerui/makerui.go | 8 +------- internal/makerui/reveal_other.go | 10 ++++++++++ internal/makerui/reveal_windows.go | 16 ++++++++++++++++ 7 files changed, 49 insertions(+), 23 deletions(-) create mode 100644 internal/makerui/reveal_other.go create mode 100644 internal/makerui/reveal_windows.go diff --git a/README.md b/README.md index 1909597..7e7da60 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ The original Zero W gets 32-bit Raspberry Pi OS with an ARMv6 build of squirreld ### Radxa ROCK 4C+ -The ROCK 4C+ gets [Armbian](https://www.armbian.com/) (Debian trixie, minimal). There is no stable Armbian release for this board, only community builds; Squirrel Maker uses one that was checked. Armbian keeps everything on ext4, which macOS and Windows cannot write, so Squirrel Maker writes its files itself, without mounting the card: it only touches a file system that is clean and uses ext4 features it knows, and reads everything back afterwards. Updating such a card asks for your password on a Mac. At the end of the first start the board restarts once; the car then sees the drive. Ethernet stays with Armbian's own network setup; NetworkManager handles wifi and the hotspot. Normally both USB 3 ports are host ports. The first start adds a device tree overlay (`deploy/armbian/dwc3-0-device.dts`) that puts the RK3399's first USB 3 controller (`dwc3_0`, the upper blue port) in peripheral mode; it takes effect after the restart. From then on squirreld offers the car's drive on that port through the kernel's USB gadget, as on a Pi (where `dtoverlay=dwc2` in `config.txt` does the same for the port marked USB). +The ROCK 4C+ gets [Armbian](https://www.armbian.com/) (Debian trixie, minimal). There is no stable Armbian release for this board, only community builds; Squirrel Maker uses one that was checked. Armbian keeps everything on ext4, which macOS and Windows cannot write, so Squirrel Maker writes its files itself, without mounting the card: it only touches a file system that is clean and uses ext4 features it knows, and reads everything back afterwards. Updating such a card asks for your password on a Mac; on Windows it uses the administrator rights the app already has, without asking. At the end of the first start the board restarts once; the car then sees the drive. Ethernet stays with Armbian's own network setup; NetworkManager handles wifi and the hotspot. Normally both USB 3 ports are host ports. The first start adds a device tree overlay (`deploy/armbian/dwc3-0-device.dts`) that puts the RK3399's first USB 3 controller (`dwc3_0`, the upper blue port) in peripheral mode; it takes effect after the restart. From then on squirreld offers the car's drive on that port through the kernel's USB gadget, as on a Pi (where `dtoverlay=dwc2` in `config.txt` does the same for the port marked USB). ### Clips on a USB drive diff --git a/internal/ext4/ext4.go b/internal/ext4/ext4.go index 9664bea..e107974 100644 --- a/internal/ext4/ext4.go +++ b/internal/ext4/ext4.go @@ -4,8 +4,10 @@ // directories and symlinks), for the features that root has. // // It reads and writes whole blocks only, which raw disks require, and -// keeps every change in memory until Flush, so an error halfway leaves -// the file system as it was. It refuses a file system it does not fully +// keeps every change in memory until Flush, so an error before Flush +// leaves the file system as it was. Flush writes the changed blocks in no +// particular order: if a write fails or is interrupted, part of the +// change can be on disk. It refuses a file system it does not fully // understand (an unknown feature) or that is not clean (a journal to // replay, errors, orphans), rather than guess. package ext4 diff --git a/internal/ext4/write.go b/internal/ext4/write.go index 5344588..0313398 100644 --- a/internal/ext4/write.go +++ b/internal/ext4/write.go @@ -822,7 +822,8 @@ func (f *FS) Remove(p string) error { } // Flush brings the checksums of the changed groups and the superblock up -// to date and writes every changed block. +// to date and writes every changed block, in no particular order; when a +// write fails, the blocks written before it stay written. func (f *FS) Flush() error { for g := range f.touched { d, n, err := f.desc(g) diff --git a/internal/maker/disks_windows.go b/internal/maker/disks_windows.go index 05e4976..b88904d 100644 --- a/internal/maker/disks_windows.go +++ b/internal/maker/disks_windows.go @@ -126,18 +126,23 @@ func checkDisk(ctx context.Context, id string) (Disk, error) { if err := diskOK(d); err != nil { return Disk{}, fmt.Errorf("%s %w", id, err) } - name := d.Name - if parts, err := partitions(ctx, n); err == nil { - for _, p := range parts { - if p.Label != "" { - name = p.Label + ", " + name - break - } + return toDisk(ctx, d), nil + } + return Disk{}, fmt.Errorf("%s not found", id) +} + +// toDisk names a disk after its first labelled partition, if it has one. +func toDisk(ctx context.Context, d winDisk) Disk { + name := d.Name + if parts, err := partitions(ctx, d.Number); err == nil { + for _, p := range parts { + if p.Label != "" { + name = p.Label + ", " + name + break } } - return Disk{ID: id, Name: name, Size: d.Size, Protocol: d.Bus}, nil } - return Disk{}, fmt.Errorf("%s not found", id) + return Disk{ID: fmt.Sprintf("PhysicalDrive%d", d.Number), Name: name, Size: d.Size, Protocol: d.Bus} } func diskOK(d winDisk) error { @@ -165,9 +170,7 @@ func ListDisks(ctx context.Context) ([]Disk, error) { var out []Disk for _, d := range disks { if diskOK(d) == nil { - if disk, err := checkDisk(ctx, fmt.Sprintf("PhysicalDrive%d", d.Number)); err == nil { - out = append(out, disk) - } + out = append(out, toDisk(ctx, d)) } } return out, nil diff --git a/internal/makerui/makerui.go b/internal/makerui/makerui.go index b1befc1..b77f461 100644 --- a/internal/makerui/makerui.go +++ b/internal/makerui/makerui.go @@ -13,7 +13,6 @@ import ( "io/fs" "net/http" "os" - "os/exec" "os/user" "path/filepath" "regexp" @@ -480,12 +479,7 @@ func (a *Server) reveal(w http.ResponseWriter, r *http.Request) { image := a.image a.mu.Unlock() if image != "" { - // Finder or Explorer shows the image selected. - if runtime.GOOS == "windows" { - exec.Command("explorer", "/select,"+image).Run() - } else { - exec.Command("open", "-R", image).Run() - } + revealFile(image) // Finder or Explorer shows the image selected } writeJSON(w, map[string]bool{"ok": true}) } diff --git a/internal/makerui/reveal_other.go b/internal/makerui/reveal_other.go new file mode 100644 index 0000000..5443f9a --- /dev/null +++ b/internal/makerui/reveal_other.go @@ -0,0 +1,10 @@ +//go:build !windows + +package makerui + +import "os/exec" + +// revealFile shows path selected in the Finder. +func revealFile(path string) { + exec.Command("open", "-R", path).Run() +} diff --git a/internal/makerui/reveal_windows.go b/internal/makerui/reveal_windows.go new file mode 100644 index 0000000..665f911 --- /dev/null +++ b/internal/makerui/reveal_windows.go @@ -0,0 +1,16 @@ +package makerui + +import ( + "os/exec" + "syscall" +) + +// revealFile shows path selected in Explorer. Explorer reads its own +// command line: Go would quote "/select,C:\a b\c" as a whole when the path +// has a space, which Explorer does not understand, so only the path is +// quoted. +func revealFile(path string) { + cmd := exec.Command("explorer.exe") + cmd.SysProcAttr = &syscall.SysProcAttr{CmdLine: `explorer.exe /select,"` + path + `"`} + cmd.Run() +} From fd1ae45ebb3f555acb28a5b3fa632f55ec68bf85 Mon Sep 17 00:00:00 2001 From: Marcus Aram <4435760+marcusaram@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:23:12 +0100 Subject: [PATCH 8/8] README: what to do when the Windows app does not start (WebView2) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e7da60..a3d7e74 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Tesla puts speed, pedals, steering and position into every frame from firmware 2 Squirrel Maker is the app that makes the SD card, for macOS and Windows. Download it from the [releases](https://github.com/marcusaram/squirrel/releases), or build it yourself (see [Development](#development)): - **macOS:** `Squirrel-Maker-.zip`, with the app inside. It is not signed with an Apple developer certificate: the first time, macOS says it cannot check the developer. Open **System Settings → Privacy & Security** and choose **Open Anyway** below the message, or right-click the app and choose **Open**. -- **Windows:** `Squirrel-Maker--windows.zip`. Unpack it and start `Squirrel Maker.exe`. It asks for administrator rights, which writing a card needs. It is not signed either: if Windows says it protected your PC, choose **More info** and then **Run anyway**. The app uses Microsoft Edge WebView2, which Windows 10 and 11 normally have. +- **Windows:** `Squirrel-Maker--windows.zip`. Unpack it and start `Squirrel Maker.exe`. It asks for administrator rights, which writing a card needs. It is not signed either: if Windows says it protected your PC, choose **More info** and then **Run anyway**. The app uses Microsoft Edge WebView2, which Windows 10 and 11 normally have. If the app does not start, install the [WebView2 Runtime](https://developer.microsoft.com/microsoft-edge/webview2/) from Microsoft, or use `squirrel-maker.exe` instead, which shows the same steps in your browser (see below). Squirrel Maker also runs from the command line (`squirrel-maker`, on Windows `squirrel-maker.exe` from a terminal opened as administrator), with the same steps in your browser.