From 1ed10f5493a9743d508cb8413b0a3bdd9dc78fa2 Mon Sep 17 00:00:00 2001 From: Gary Date: Mon, 9 Nov 2026 11:21:44 +0100 Subject: [PATCH 1/2] feat(zfs): read ZFS pools, datasets and snapshots (read only) GET /v1/disks/zfs, /v1/disks/zfs/snapshots and /v1/disks/zfs/io. zpool list, zpool status and zfs list run in their scripted forms, without a shell, with time-outs and one at a time; a box without ZFS answers available: false. The test fixtures are written from the OpenZFS documentation and a pasted healthy pool, not captured from a box. --- CHANGELOG.md | 1 + route/v1.go | 3 + route/v1/zfs.go | 43 ++ route/v1/zfs_test.go | 67 ++ service/testdata/zfs/README.md | 8 + service/testdata/zfs/kstat_io.txt | 3 + service/testdata/zfs/zfs_list_datasets.txt | 6 + service/testdata/zfs/zfs_list_none.txt | 1 + service/testdata/zfs/zfs_list_root_only.txt | 1 + service/testdata/zfs/zfs_list_snapshots.txt | 2 + service/testdata/zfs/zpool_list_none.txt | 1 + service/testdata/zfs/zpool_list_one.txt | 1 + service/testdata/zfs/zpool_list_two.txt | 2 + .../zfs/zpool_status_degraded_raidz.txt | 19 + service/testdata/zfs/zpool_status_groups.txt | 21 + .../zfs/zpool_status_healthy_mirror.txt | 12 + .../zfs/zpool_status_scrub_running.txt | 12 + service/zfs.go | 708 ++++++++++++++++++ service/zfs_test.go | 497 ++++++++++++ 19 files changed, 1408 insertions(+) create mode 100644 route/v1/zfs.go create mode 100644 route/v1/zfs_test.go create mode 100644 service/testdata/zfs/README.md create mode 100644 service/testdata/zfs/kstat_io.txt create mode 100644 service/testdata/zfs/zfs_list_datasets.txt create mode 100644 service/testdata/zfs/zfs_list_none.txt create mode 100644 service/testdata/zfs/zfs_list_root_only.txt create mode 100644 service/testdata/zfs/zfs_list_snapshots.txt create mode 100644 service/testdata/zfs/zpool_list_none.txt create mode 100644 service/testdata/zfs/zpool_list_one.txt create mode 100644 service/testdata/zfs/zpool_list_two.txt create mode 100644 service/testdata/zfs/zpool_status_degraded_raidz.txt create mode 100644 service/testdata/zfs/zpool_status_groups.txt create mode 100644 service/testdata/zfs/zpool_status_healthy_mirror.txt create mode 100644 service/testdata/zfs/zpool_status_scrub_running.txt create mode 100644 service/zfs.go create mode 100644 service/zfs_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 80b24d4..c354263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to CasaOS LocalStorage are documented here. ### Added +- **ZFS pools, datasets and snapshots, read only** (`GET /v1/disks/zfs`, `/v1/disks/zfs/snapshots?dataset=`, `/v1/disks/zfs/io`). Pools (size, allocated, free, fragmentation, health), the state of each (`zpool status`: scan, errors, tree of disks), datasets, the snapshots of one dataset (newest 200), and the I/O counters of each pool read from the kernel. Only `zpool list`, `zpool status`, `zfs list` run, without a shell, in their scripted forms (`-H -p`), with time-outs, one at a time: a pool that stopped answering cannot pile processes up. A box without ZFS answers `available: false`. Nothing here changes a pool. - **`CASAOS_LSBLK_FILE`** (development): when set, `ExecLSBLK` returns that file instead of running host `lsblk`. `ExecLSBLKByPath` filters the same document. - **`CASAOS_FAKE_STORAGE=1`** (development): `mount`, `umount`, `parted`, `partprobe`, `e2label` and `sfdisk` return success without exec, only when `CASAOS_LSBLK_FILE` is set too. `mdadm`, `mkfs` and `wipefs` still exec. Production is unchanged when the variables are unset. diff --git a/route/v1.go b/route/v1.go index 19a931b..6f32bbb 100644 --- a/route/v1.go +++ b/route/v1.go @@ -78,6 +78,9 @@ func InitV1Router() http.Handler { v1DisksGroup.DELETE("/snapraid", v1.DeleteSnapRAID) v1DisksGroup.DELETE("/snapraid/job", v1.DeleteSnapRAIDJob) v1DisksGroup.POST("/snapraid/:action", v1.PostSnapRAIDAction) + v1DisksGroup.GET("/zfs", v1.GetZFS) + v1DisksGroup.GET("/zfs/snapshots", v1.GetZFSSnapshots) + v1DisksGroup.GET("/zfs/io", v1.GetZFSIO) } v1StorageGroup := v1Group.Group("/storage") diff --git a/route/v1/zfs.go b/route/v1/zfs.go new file mode 100644 index 0000000..8eef94f --- /dev/null +++ b/route/v1/zfs.go @@ -0,0 +1,43 @@ +package v1 + +import ( + "errors" + "net/http" + + "github.com/ReCasaOS/CasaOS-Common/model" + "github.com/ReCasaOS/CasaOS-Common/utils/common_err" + "github.com/ReCasaOS/CasaOS-LocalStorage/service" + "github.com/labstack/echo/v4" +) + +// zfsKstat is where the kernel keeps a folder of counters for each pool, a var for the tests. +var zfsKstat = "/proc/spl/kstat/zfs" + +// GetZFS is the ZFS pools and datasets, read only. A box without ZFS answers 200 with available false and the reason, so that the dashboard keeps quiet. +func GetZFS(ctx echo.Context) error { + return ctx.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: service.ZFSOverview()}) +} + +// GetZFSSnapshots lists the snapshots of the dataset in ?dataset=, the newest first. A dataset the box does not list is 404 and nothing is run for it. +func GetZFSSnapshots(ctx echo.Context) error { + list, err := service.ZFSSnapshotsOf(ctx.QueryParam("dataset")) + if err != nil { + status := http.StatusInternalServerError + switch { + case errors.Is(err, service.ErrZFSNoDataset): + status = http.StatusNotFound + case errors.Is(err, service.ErrZFSUnavailable): + status = http.StatusConflict + case errors.Is(err, service.ErrZFSBusy): + status = http.StatusServiceUnavailable + } + return ctx.JSON(status, model.Result{Success: common_err.SERVICE_ERROR, Message: err.Error(), Data: err.Error()}) + } + return ctx.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: list}) +} + +// GetZFSIO is the counters of every pool (bytes and operations since it was imported), read from the kernel without running anything: the rate is the +// difference between two readings. +func GetZFSIO(ctx echo.Context) error { + return ctx.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: service.ZFSIO(zfsKstat)}) +} diff --git a/route/v1/zfs_test.go b/route/v1/zfs_test.go new file mode 100644 index 0000000..18442f3 --- /dev/null +++ b/route/v1/zfs_test.go @@ -0,0 +1,67 @@ +package v1 + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/labstack/echo/v4" +) + +func zfsCall(t *testing.T, handler echo.HandlerFunc, target string) (int, map[string]interface{}) { + t.Helper() + rec := httptest.NewRecorder() + c := echo.New().NewContext(httptest.NewRequest(http.MethodGet, target, nil), rec) + if err := handler(c); err != nil { + t.Fatal(err) + } + var answer map[string]interface{} + if err := json.Unmarshal(rec.Body.Bytes(), &answer); err != nil { + t.Fatalf("%s: %v", rec.Body.String(), err) + } + return rec.Code, answer +} + +// On the machine the tests run on there is no ZFS: the answer is a quiet 200, not an error. +func TestGetZFSWithoutZFS(t *testing.T) { + code, answer := zfsCall(t, GetZFS, "/v1/disks/zfs") + data, _ := answer["data"].(map[string]interface{}) + if code != http.StatusOK || data["available"] != false || data["reason"] == "" { + t.Fatalf("%d %v", code, answer) + } + if pools, ok := data["pools"].([]interface{}); !ok || len(pools) != 0 { + t.Fatalf("pools must be an empty list, not null: %v", data["pools"]) + } +} + +func TestGetZFSSnapshotsOfADatasetThatIsNotThere(t *testing.T) { + code, _ := zfsCall(t, GetZFSSnapshots, "/v1/disks/zfs/snapshots?dataset=-r") + if code != http.StatusConflict && code != http.StatusNotFound { + t.Fatalf("%d", code) + } +} + +func TestGetZFSIO(t *testing.T) { + dir := t.TempDir() + saved := zfsKstat + zfsKstat = dir + t.Cleanup(func() { zfsKstat = saved }) + if err := os.MkdirAll(filepath.Join(dir, "tank"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "tank", "io"), []byte("1 1 0x01 4 1 1 1\nnread nwritten reads writes\n10 20 3 4\n"), 0o644); err != nil { + t.Fatal(err) + } + code, answer := zfsCall(t, GetZFSIO, "/v1/disks/zfs/io") + list, _ := answer["data"].([]interface{}) + if code != http.StatusOK || len(list) != 1 { + t.Fatalf("%d %v", code, answer) + } + pool := list[0].(map[string]interface{}) + if pool["pool"] != "tank" || pool["read_bytes"] != float64(10) || pool["write_ops"] != float64(4) { + t.Fatalf("%v", pool) + } +} diff --git a/service/testdata/zfs/README.md b/service/testdata/zfs/README.md new file mode 100644 index 0000000..2c8af67 --- /dev/null +++ b/service/testdata/zfs/README.md @@ -0,0 +1,8 @@ +# ZFS fixtures + +The outputs of `zpool` and `zfs` the parsers in `service/zfs.go` are tested on. + +**These files were written from the OpenZFS documentation and man pages, and from the output a user pasted in +ReCasaOS/CasaOS#66 (the healthy mirror), not captured from a box.** Replace them with real captures when there are some +(`zpool list -H -p -o ...`, `zpool status -p -P `, `zfs list -H -p ...`, on OpenZFS 2.1, 2.2 and 2.3), and take this +note out then. diff --git a/service/testdata/zfs/kstat_io.txt b/service/testdata/zfs/kstat_io.txt new file mode 100644 index 0000000..65975c3 --- /dev/null +++ b/service/testdata/zfs/kstat_io.txt @@ -0,0 +1,3 @@ +23 1 0x01 4 1088 21204339 64847143498 +nread nwritten reads writes wtime wlentime wupdate rtime rlentime rupdate wcnt rcnt +9990000000 7020000000 1320000 1740000 0 0 0 0 0 0 0 0 diff --git a/service/testdata/zfs/zfs_list_datasets.txt b/service/testdata/zfs/zfs_list_datasets.txt new file mode 100644 index 0000000..544fdfe --- /dev/null +++ b/service/testdata/zfs/zfs_list_datasets.txt @@ -0,0 +1,6 @@ +pool1 filesystem 1264462766080 871000000000 98304 0 /pool1 yes 0 - +pool1/bitcoin-data filesystem 759000000000 671000000000 759000000000 0 /pool1/bitcoin-data yes 0 - +pool1/appdata filesystem 300647710720 671000000000 300647710720 31457280 /pool1/appdata yes 0 - +pool1/immich-db filesystem 1057000000 671000000000 1055000000 655360 /pool1/immich-db yes 0 - +pool1/vm-disk volume 107374182400 770000000000 12000000000 0 - - 0 - +pool1/secret filesystem 98304 671000000000 98304 0 /pool1/secret no 1073741824 unavailable diff --git a/service/testdata/zfs/zfs_list_none.txt b/service/testdata/zfs/zfs_list_none.txt new file mode 100644 index 0000000..96afae2 --- /dev/null +++ b/service/testdata/zfs/zfs_list_none.txt @@ -0,0 +1 @@ +no datasets available diff --git a/service/testdata/zfs/zfs_list_root_only.txt b/service/testdata/zfs/zfs_list_root_only.txt new file mode 100644 index 0000000..e58dd65 --- /dev/null +++ b/service/testdata/zfs/zfs_list_root_only.txt @@ -0,0 +1 @@ +pool1 filesystem 1264462766080 871000000000 98304 0 /pool1 yes 0 - diff --git a/service/testdata/zfs/zfs_list_snapshots.txt b/service/testdata/zfs/zfs_list_snapshots.txt new file mode 100644 index 0000000..6e9409b --- /dev/null +++ b/service/testdata/zfs/zfs_list_snapshots.txt @@ -0,0 +1,2 @@ +pool1/appdata@backup_20261105_023008 12373196 299000000000 17000000 1762309808 +pool1/appdata@backup_20261105_013917 20447232 299000000000 299000000000 1762306757 diff --git a/service/testdata/zfs/zpool_list_none.txt b/service/testdata/zfs/zpool_list_none.txt new file mode 100644 index 0000000..f42284d --- /dev/null +++ b/service/testdata/zfs/zpool_list_none.txt @@ -0,0 +1 @@ +no pools available diff --git a/service/testdata/zfs/zpool_list_one.txt b/service/testdata/zfs/zpool_list_one.txt new file mode 100644 index 0000000..291d4d3 --- /dev/null +++ b/service/testdata/zfs/zpool_list_one.txt @@ -0,0 +1 @@ +pool1 1992864825344 1060842668032 932022157312 29 53 ONLINE diff --git a/service/testdata/zfs/zpool_list_two.txt b/service/testdata/zfs/zpool_list_two.txt new file mode 100644 index 0000000..c9e40b6 --- /dev/null +++ b/service/testdata/zfs/zpool_list_two.txt @@ -0,0 +1,2 @@ +pool1 1992864825344 1060842668032 932022157312 29 53 ONLINE +backup 3985729650688 3500000000000 485729650688 - 87 DEGRADED diff --git a/service/testdata/zfs/zpool_status_degraded_raidz.txt b/service/testdata/zfs/zpool_status_degraded_raidz.txt new file mode 100644 index 0000000..ec92bd2 --- /dev/null +++ b/service/testdata/zfs/zpool_status_degraded_raidz.txt @@ -0,0 +1,19 @@ + pool: backup + state: DEGRADED +status: One or more devices could not be opened. Sufficient replicas exist for + the pool to continue functioning in a degraded state. +action: Attach the missing device and online it using 'zpool online'. + see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-2Q + scan: resilver in progress since Sun Oct 11 03:10:00 2026 + 1.20T scanned at 1.1G/s, 800G issued at 700M/s, 1.80T total + 300G resilvered, 43.20% done, 00:20:11 to go +config: + + NAME STATE READ WRITE CKSUM + backup DEGRADED 0 0 0 + raidz1-0 DEGRADED 0 0 0 + /dev/sda1 ONLINE 0 0 0 + /dev/sdb1 ONLINE 0 2 0 + 11485761428236503841 UNAVAIL 0 0 0 was /dev/sdc1 + +errors: 2 data errors, use '-v' for a list diff --git a/service/testdata/zfs/zpool_status_groups.txt b/service/testdata/zfs/zpool_status_groups.txt new file mode 100644 index 0000000..16c6584 --- /dev/null +++ b/service/testdata/zfs/zpool_status_groups.txt @@ -0,0 +1,21 @@ + pool: tank + state: ONLINE + scan: scrub canceled on Sun Oct 11 01:02:03 2026 +config: + + NAME STATE READ WRITE CKSUM + tank ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + /dev/sda ONLINE 0 0 0 + /dev/sdb ONLINE 0 0 0 + mirror-1 ONLINE 0 0 0 + /dev/sdc ONLINE 0 0 0 + /dev/sdd ONLINE 0 0 0 + logs + /dev/sde ONLINE 0 0 0 + cache + /dev/sdf ONLINE 0 0 0 + spares + /dev/sdg AVAIL + +errors: No known data errors diff --git a/service/testdata/zfs/zpool_status_healthy_mirror.txt b/service/testdata/zfs/zpool_status_healthy_mirror.txt new file mode 100644 index 0000000..0bcb178 --- /dev/null +++ b/service/testdata/zfs/zpool_status_healthy_mirror.txt @@ -0,0 +1,12 @@ + pool: pool1 + state: ONLINE + scan: scrub repaired 0B in 00:16:47 with 0 errors on Sun Oct 11 00:40:48 2026 +config: + + NAME STATE READ WRITE CKSUM + pool1 ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + /dev/disk/by-id/nvme-WD_BLACK_SN770_2TB_24443A806391_1-part1 ONLINE 0 0 0 + /dev/disk/by-id/nvme-WD_BLACK_SN770_2TB_24443A805750_1-part1 ONLINE 0 0 0 + +errors: No known data errors diff --git a/service/testdata/zfs/zpool_status_scrub_running.txt b/service/testdata/zfs/zpool_status_scrub_running.txt new file mode 100644 index 0000000..118e80b --- /dev/null +++ b/service/testdata/zfs/zpool_status_scrub_running.txt @@ -0,0 +1,12 @@ + pool: tank + state: ONLINE + scan: scrub in progress since Sun Oct 11 00:24:00 2026 + 500G scanned at 1.0G/s, 400G issued at 800M/s, 1.80T total + 0B repaired, 21.70% done, no estimated completion time +config: + + NAME STATE READ WRITE CKSUM + tank ONLINE 0 0 0 + /dev/sda ONLINE 0 0 0 + +errors: No known data errors diff --git a/service/zfs.go b/service/zfs.go new file mode 100644 index 0000000..93243cd --- /dev/null +++ b/service/zfs.go @@ -0,0 +1,708 @@ +package service + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// ZFS, read only: the pools, their datasets and snapshots, as `zpool` and `zfs` print them. Nothing here changes a pool: the commands below are the only ones +// that are ever run, as an argument list (no shell), and the one argument a client gives (a dataset) must be one the box has just listed. +// +// The numbers come from the scripted forms (-H, tab separated, -p, exact bytes), never the forms made for people. `zpool status` has no -H: it is read by its +// section names, and a line that is not understood is skipped. + +var ( + // ErrZFSNoDataset: the dataset is not one the box lists. + ErrZFSNoDataset = errors.New("no such ZFS dataset") + // ErrZFSUnavailable: ZFS is not usable on this box. + ErrZFSUnavailable = errors.New("ZFS is not available") + // ErrZFSBusy: a ZFS command is still running (or stuck), none is started on top of it. + ErrZFSBusy = errors.New("a ZFS command is still running") +) + +const ( + zfsCommandTimeout = 10 * time.Second + zfsSnapshotTimeout = 20 * time.Second + zfsRequestTimeout = 30 * time.Second + zfsListTTL = 10 * time.Second + zfsMaxOutput = 4 << 20 + // ZFSSnapshotLimit is how many snapshots of one dataset an answer holds, the newest first. + ZFSSnapshotLimit = 200 +) + +// ZFSState is what GET /v1/disks/zfs answers. +type ZFSState struct { + // Available: ZFS can be used here. Reason says why not when it is false: tools_missing (zpool or zfs not installed), module_missing (installed, not + // loaded in the kernel) or failed (a command failed otherwise). + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` + // Busy: a command is still running, there is nothing to show yet. Stale: this is the last answer that was got, which is older than it should be. + Busy bool `json:"busy,omitempty"` + Stale bool `json:"stale,omitempty"` + Pools []ZFSPool `json:"pools"` + // DatasetsKnown is false when the datasets could not be read: an empty list then means nothing. + DatasetsKnown bool `json:"datasets_known"` + Datasets []ZFSDataset `json:"datasets"` +} + +// ZFSPool is a pool; all sizes are bytes, -1 stands for "not said". +type ZFSPool struct { + Name string `json:"name"` + Health string `json:"health"` + Size int64 `json:"size"` + Allocated int64 `json:"allocated"` + Free int64 `json:"free"` + Fragmentation int `json:"fragmentation"` // percent + Capacity int `json:"capacity"` // percent + // Known: `zpool status` could be read. When it is false only the figures above are good, and the pool must not be shown as healthy. + Known bool `json:"known"` + // State is the pool's state in `zpool status`; Status, Action and See are its own words (English, ZFS does not translate them). + State string `json:"state"` + Status string `json:"status,omitempty"` + Action string `json:"action,omitempty"` + See string `json:"see,omitempty"` + // DataErrors is the number of files with a permanent error, -1 when the line was not read. + DataErrors int `json:"data_errors"` + Scan ZFSScan `json:"scan"` + Vdevs []ZFSVdev `json:"vdevs"` +} + +// ZFSScan is the last scan (scrub or resilver); ZFS keeps one only, a resilver wipes the memory of the scrub before it. +type ZFSScan struct { + // Kind: none, scrub, resilver or other (Raw says it). + Kind string `json:"kind"` + // State: finished, running, paused, canceled, or empty. + State string `json:"state,omitempty"` + // Percent and Remaining (seconds) of a scan at work, -1 when not said. + Percent float64 `json:"percent"` + Remaining int64 `json:"remaining"` + // Repaired is how much was repaired, as ZFS wrote it ("0B"). + Repaired string `json:"repaired,omitempty"` + Errors int `json:"errors"` // -1 when not said + Duration int64 `json:"duration"` // seconds, -1 when not said + // Started and Ended are Unix seconds (the box's local time, which the line does not name), 0 when not said. + Started int64 `json:"started"` + Ended int64 `json:"ended"` + Raw string `json:"raw,omitempty"` +} + +// ZFSVdev is a line of the pool's tree: a mirror, a raidz, a disk, or a group (logs, cache, spares, special, dedup). +type ZFSVdev struct { + Name string `json:"name"` + // Kind: mirror, raidz, draid, replacing, spare, group or disk. + Kind string `json:"kind"` + State string `json:"state,omitempty"` + Read int64 `json:"read"` // error counters, -1 when not said + Write int64 `json:"write"` + Cksum int64 `json:"cksum"` + Note string `json:"note,omitempty"` + // Disk is the disk a disk vdev is on (sdb, nvme0n1), when it can be told. + Disk string `json:"disk,omitempty"` + Children []ZFSVdev `json:"children,omitempty"` +} + +// ZFSDataset is a filesystem or a volume. +type ZFSDataset struct { + Name string `json:"name"` + Pool string `json:"pool"` + // Type: filesystem or volume. + Type string `json:"type"` + Used int64 `json:"used"` // with its snapshots and its children + Available int64 `json:"available"` + Referenced int64 `json:"referenced"` + UsedBySnapshots int64 `json:"used_by_snapshots"` + Mountpoint string `json:"mountpoint"` // a path, none or legacy; empty for a volume + Mounted bool `json:"mounted"` + Quota int64 `json:"quota"` // 0: none + // KeyStatus of an encrypted dataset: available or unavailable; empty when it is not encrypted. + KeyStatus string `json:"key_status,omitempty"` +} + +// ZFSSnapshot is a snapshot of a dataset; Name is the part after the @. +type ZFSSnapshot struct { + Name string `json:"name"` + Used int64 `json:"used"` + Referenced int64 `json:"referenced"` + Written int64 `json:"written"` + Created int64 `json:"created"` // Unix seconds +} + +// ZFSSnapshots is the answer to GET /v1/disks/zfs/snapshots. +type ZFSSnapshots struct { + Dataset string `json:"dataset"` + Snapshots []ZFSSnapshot `json:"snapshots"` + // Truncated: there are more than ZFSSnapshotLimit; these are the newest. + Truncated bool `json:"truncated"` +} + +// ZFSPoolIO holds the counters of a pool since it was imported; the rate is the difference between two readings over the time between them. +type ZFSPoolIO struct { + Pool string `json:"pool"` + ReadBytes int64 `json:"read_bytes"` + WriteBytes int64 `json:"write_bytes"` + ReadOps int64 `json:"read_ops"` + WriteOps int64 `json:"write_ops"` + // At is when it was read, Unix milliseconds on the box. + At int64 `json:"at"` +} + +// zfsHas says whether the tools are installed, a var for the tests. +var zfsHas = func() bool { + for _, tool := range []string{"zpool", "zfs"} { + if _, err := exec.LookPath(tool); err != nil { + return false + } + } + return true +} + +// zfsExec runs a command and answers its output. A var for the tests. The output is read up to zfsMaxOutput. +var zfsExec = func(ctx context.Context, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = append(os.Environ(), "LC_ALL=C") // the words of the output are English whatever the language of the box + cmd.WaitDelay = 2 * time.Second + out, stderr := &cappedBuffer{max: zfsMaxOutput}, &cappedBuffer{max: 4096} + cmd.Stdout, cmd.Stderr = out, stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String())) + } + return out.Bytes(), nil +} + +// cappedBuffer keeps the first max bytes and drops the rest, so a pool with a million snapshots cannot fill the memory of a small box. +type cappedBuffer struct { + bytes.Buffer + max int +} + +func (b *cappedBuffer) Write(p []byte) (int, error) { + if room := b.max - b.Len(); room > 0 { + b.Buffer.Write(p[:min(room, len(p))]) + } + return len(p), nil +} + +// zfsSlot lets one command run at a time. It is given back when the command really ends, not when its caller gives up: a pool that stopped answering can leave +// `zpool status` stuck where it cannot be killed, and a dashboard that asks every few seconds must not pile such processes up. +var zfsSlot = make(chan struct{}, 1) + +// zfsKillGrace is how long a command that was told to stop is waited for, a var for the tests. +var zfsKillGrace = 3 * time.Second + +func runZFS(ctx context.Context, timeout time.Duration, name string, args ...string) ([]byte, error) { + select { + case zfsSlot <- struct{}{}: + case <-ctx.Done(): + return nil, ErrZFSBusy + } + cmdCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + type result struct { + out []byte + err error + } + done := make(chan result, 1) + go func() { + out, err := zfsExec(cmdCtx, name, args...) + <-zfsSlot + done <- result{out, err} + }() + select { + case r := <-done: + return r.out, r.err + case <-cmdCtx.Done(): + // the process gets its kill and a moment to go; if it does not, we stop waiting and the slot stays taken until it does + select { + case r := <-done: + if r.err == nil { + return r.out, nil + } + case <-time.After(zfsKillGrace): + } + return nil, fmt.Errorf("%s: %w", name, cmdCtx.Err()) + } +} + +var ( + zfsCache struct { + sync.Mutex + state *ZFSState + at time.Time + flight chan struct{} + } + zfsNow = time.Now +) + +// ZFSOverview is what the dashboard shows: the pools and the datasets. It is kept for zfsListTTL, and shared by whoever asks meanwhile; when the commands +// cannot be run (one is still stuck), the last answer is given back marked stale, or Busy if there is none. +func ZFSOverview() ZFSState { + c := &zfsCache + c.Lock() + if c.state != nil && zfsNow().Sub(c.at) < zfsListTTL { + s := *c.state + c.Unlock() + return s + } + if c.flight != nil { + flight := c.flight + c.Unlock() + select { + case <-flight: + case <-time.After(zfsRequestTimeout): + } + c.Lock() + defer c.Unlock() + return zfsLastKnown() + } + c.flight = make(chan struct{}) + flight := c.flight + c.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), zfsRequestTimeout) + state := gatherZFS(ctx) + cancel() + + c.Lock() + defer c.Unlock() + if state.Busy || state.Reason == "failed" { + state = zfsLastKnown() + } + s := state + c.state, c.at = &s, zfsNow() + c.flight = nil + close(flight) + return state +} + +// zfsLastKnown is the last good answer, marked stale; call it with the cache locked. +func zfsLastKnown() ZFSState { + if zfsCache.state == nil || zfsCache.state.Busy { + return ZFSState{Available: true, Busy: true, Pools: []ZFSPool{}, Datasets: []ZFSDataset{}} + } + s := *zfsCache.state + s.Stale = zfsNow().Sub(zfsCache.at) >= zfsListTTL + return s +} + +func gatherZFS(ctx context.Context) ZFSState { + state := ZFSState{Pools: []ZFSPool{}, Datasets: []ZFSDataset{}} + if !zfsHas() { + state.Reason = "tools_missing" + return state + } + out, err := runZFS(ctx, zfsCommandTimeout, "zpool", "list", "-H", "-p", "-o", "name,size,allocated,free,fragmentation,capacity,health") + switch { + case errors.Is(err, ErrZFSBusy): + state.Available, state.Busy = true, true + return state + case err != nil && (strings.Contains(err.Error(), "modules are not loaded") || strings.Contains(err.Error(), "/dev/zfs")): + state.Reason = "module_missing" + return state + case err != nil: + state.Reason = "failed" + return state + } + state.Available = true + state.Pools = parseZpoolList(string(out)) + if len(state.Pools) == 0 { + return state + } + for i := range state.Pools { + pool := &state.Pools[i] + text, err := runZFS(ctx, zfsCommandTimeout, "zpool", "status", "-p", "-P", pool.Name) + if err != nil { + continue // Known stays false: the pool is not shown as well + } + parseZpoolStatus(string(text), pool) + } + text, err := runZFS(ctx, zfsCommandTimeout, "zfs", "list", "-H", "-p", "-t", "filesystem,volume", "-o", "name,type,used,available,referenced,usedbysnapshots,mountpoint,mounted,quota,keystatus") + if err == nil { + state.DatasetsKnown = true + state.Datasets = parseZfsList(string(text)) + } + return state +} + +// ZFSSnapshotsOf lists the snapshots of one dataset, the newest first. The dataset must be one the box lists. +func ZFSSnapshotsOf(dataset string) (ZFSSnapshots, error) { + overview := ZFSOverview() + if !overview.Available { + return ZFSSnapshots{}, ErrZFSUnavailable + } + if overview.Busy { + return ZFSSnapshots{}, ErrZFSBusy + } + known := false + for _, d := range overview.Datasets { + known = known || d.Name == dataset + } + if !known || strings.HasPrefix(dataset, "-") { + return ZFSSnapshots{}, ErrZFSNoDataset + } + ctx, cancel := context.WithTimeout(context.Background(), zfsRequestTimeout) + defer cancel() + out, err := runZFS(ctx, zfsSnapshotTimeout, "zfs", "list", "-H", "-p", "-t", "snapshot", "-d", "1", "-S", "creation", "-o", "name,used,referenced,written,creation", dataset) + if errors.Is(err, ErrZFSBusy) { + return ZFSSnapshots{}, ErrZFSBusy + } + if err != nil { + return ZFSSnapshots{}, err + } + list := ZFSSnapshots{Dataset: dataset, Snapshots: parseZfsSnapshots(string(out))} + if len(list.Snapshots) > ZFSSnapshotLimit { + list.Snapshots, list.Truncated = list.Snapshots[:ZFSSnapshotLimit], true + } + return list, nil +} + +// ZFSIO reads the counters of every pool under the kstat folder at proc ("/proc/spl/kstat/zfs"). No command is run; a pool with no such file is left out. +func ZFSIO(proc string) []ZFSPoolIO { + files, _ := filepath.Glob(filepath.Join(proc, "*", "io")) + sort.Strings(files) + now := zfsNow().UnixMilli() + list := []ZFSPoolIO{} + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + continue + } + if io, ok := parseZfsKstatIO(string(data)); ok { + io.Pool, io.At = filepath.Base(filepath.Dir(file)), now + list = append(list, io) + } + } + return list +} + +// --- parsers: text in, values out, nothing is run --- + +func zfsInt(s string) int64 { + n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + if err != nil { + return -1 // "-" is how ZFS says a value does not apply + } + return n +} + +// parseZpoolList reads `zpool list -H -p -o name,size,allocated,free,fragmentation,capacity,health`. A line without all the fields ("no pools available") is skipped. +func parseZpoolList(text string) []ZFSPool { + pools := []ZFSPool{} + for _, line := range strings.Split(text, "\n") { + f := strings.Split(strings.TrimRight(line, "\r"), "\t") + if len(f) < 7 || f[0] == "" { + continue + } + pools = append(pools, ZFSPool{ + Name: f[0], Size: zfsInt(f[1]), Allocated: zfsInt(f[2]), Free: zfsInt(f[3]), + Fragmentation: int(zfsInt(f[4])), Capacity: int(zfsInt(f[5])), Health: f[6], + DataErrors: -1, Scan: ZFSScan{Kind: "none", Percent: -1, Remaining: -1, Errors: -1, Duration: -1}, Vdevs: []ZFSVdev{}, + }) + } + return pools +} + +// parseZfsList reads `zfs list -H -p -t filesystem,volume -o name,type,used,available,referenced,usedbysnapshots,mountpoint,mounted,quota,keystatus`. +func parseZfsList(text string) []ZFSDataset { + datasets := []ZFSDataset{} + for _, line := range strings.Split(text, "\n") { + f := strings.Split(strings.TrimRight(line, "\r"), "\t") + if len(f) < 10 || f[0] == "" { + continue + } + d := ZFSDataset{ + Name: f[0], Pool: strings.SplitN(f[0], "/", 2)[0], Type: f[1], + Used: zfsInt(f[2]), Available: zfsInt(f[3]), Referenced: zfsInt(f[4]), UsedBySnapshots: zfsInt(f[5]), + Mounted: f[7] == "yes", Quota: max(zfsInt(f[8]), 0), + } + if f[6] != "-" { + d.Mountpoint = f[6] + } + if f[9] != "-" { + d.KeyStatus = f[9] + } + datasets = append(datasets, d) + } + return datasets +} + +// parseZfsSnapshots reads `zfs list -H -p -t snapshot -o name,used,referenced,written,creation`. +func parseZfsSnapshots(text string) []ZFSSnapshot { + snapshots := []ZFSSnapshot{} + for _, line := range strings.Split(text, "\n") { + f := strings.Split(strings.TrimRight(line, "\r"), "\t") + if len(f) < 5 { + continue + } + _, short, ok := strings.Cut(f[0], "@") + if !ok { + continue + } + snapshots = append(snapshots, ZFSSnapshot{Name: short, Used: zfsInt(f[1]), Referenced: zfsInt(f[2]), Written: zfsInt(f[3]), Created: max(zfsInt(f[4]), 0)}) + } + return snapshots +} + +// parseZfsKstatIO reads /proc/spl/kstat/zfs//io: a header line, a line of column names, a line of values. +func parseZfsKstatIO(text string) (ZFSPoolIO, bool) { + lines := strings.Split(text, "\n") + for i, line := range lines { + names := strings.Fields(line) + if i+1 >= len(lines) || len(names) == 0 { + continue + } + col := map[string]int{} + for n, name := range names { + col[name] = n + } + values := strings.Fields(lines[i+1]) + get := func(name string) (int64, bool) { + n, ok := col[name] + if !ok || n >= len(values) { + return 0, false + } + v, err := strconv.ParseInt(values[n], 10, 64) + return v, err == nil + } + nread, ok1 := get("nread") + nwritten, ok2 := get("nwritten") + reads, ok3 := get("reads") + writes, ok4 := get("writes") + if ok1 && ok2 && ok3 && ok4 { + return ZFSPoolIO{ReadBytes: nread, WriteBytes: nwritten, ReadOps: reads, WriteOps: writes}, true + } + } + return ZFSPoolIO{}, false +} + +var zfsSection = regexp.MustCompile(`^ {0,3}(pool|state|status|action|see|scan|config|errors):\s?(.*)$`) + +// parseZpoolStatus fills the pool with what `zpool status -p -P ` says. It never fails: what is not understood is left as it was, and Known is set only +// when the state was read. +func parseZpoolStatus(text string, pool *ZFSPool) { + sections := map[string][]string{} + var key string + var config []string + inConfig := false + for _, line := range strings.Split(text, "\n") { + line = strings.TrimRight(line, "\r") + if m := zfsSection.FindStringSubmatch(line); m != nil { + key = m[1] + inConfig = key == "config" + if !inConfig { + sections[key] = append(sections[key], strings.TrimSpace(m[2])) + } + continue + } + switch { + case key == "" || strings.TrimSpace(line) == "": + // the blank line closes the tree, and is nothing elsewhere + if inConfig && len(config) > 0 { + inConfig = false + } + case inConfig: + config = append(config, line) + default: + sections[key] = append(sections[key], strings.TrimSpace(line)) + } + } + join := func(k string) string { return strings.TrimSpace(strings.Join(sections[k], " ")) } + + pool.State = join("state") + pool.Known = pool.State != "" + pool.Status, pool.Action, pool.See = join("status"), join("action"), join("see") + pool.DataErrors = parseZfsDataErrors(join("errors")) + pool.Scan = parseZfsScan(sections["scan"]) + pool.Vdevs = parseZfsVdevs(config) +} + +var zfsDataErrors = regexp.MustCompile(`^(\d+) data errors?`) + +func parseZfsDataErrors(line string) int { + if strings.HasPrefix(line, "No known data errors") { + return 0 + } + if m := zfsDataErrors.FindStringSubmatch(line); m != nil { + n, _ := strconv.Atoi(m[1]) + return n + } + return -1 +} + +var ( + zfsScanDone = regexp.MustCompile(`^(?:(scrub) repaired|(resilvered)) (\S+) in (.+?) with (\d+) errors? on (.+)$`) + zfsScanRunning = regexp.MustCompile(`^(scrub|resilver) in progress since (.+)$`) + zfsScanStopped = regexp.MustCompile(`^(scrub|resilver) (canceled|paused)(?: on| since) (.+)$`) + zfsScanPercent = regexp.MustCompile(`(\d+(?:\.\d+)?)% done`) + zfsScanToGo = regexp.MustCompile(`(\S+) to go`) + zfsDuration = regexp.MustCompile(`^(?:(\d+) days? )?(\d+):(\d+):(\d+)$`) +) + +// zfsTime reads "Sun Oct 11 00:40:48 2026" in the box's own time zone, which the line does not name. +func zfsTime(s string) int64 { + t, err := time.ParseInLocation("Mon Jan _2 15:04:05 2006", strings.Join(strings.Fields(s), " "), time.Local) + if err != nil { + return 0 + } + return t.Unix() +} + +func zfsDurationSeconds(s string) int64 { + m := zfsDuration.FindStringSubmatch(strings.TrimSpace(s)) + if m == nil { + return -1 + } + days, _ := strconv.Atoi(m[1]) + h, _ := strconv.Atoi(m[2]) + mi, _ := strconv.Atoi(m[3]) + sec, _ := strconv.Atoi(m[4]) + return int64(((days*24+h)*60+mi)*60 + sec) +} + +func parseZfsScan(lines []string) ZFSScan { + scan := ZFSScan{Kind: "none", Percent: -1, Remaining: -1, Errors: -1, Duration: -1} + if len(lines) == 0 || lines[0] == "" { + return scan + } + scan.Raw = strings.Join(lines, " ") + first := lines[0] + rest := strings.Join(lines[1:], " ") + switch { + case strings.HasPrefix(first, "none requested"): + scan.Kind, scan.Raw = "none", "" + case zfsScanDone.MatchString(first): + m := zfsScanDone.FindStringSubmatch(first) + scan.Kind, scan.State = "scrub", "finished" + if m[2] != "" { + scan.Kind = "resilver" + } + scan.Repaired, scan.Duration = m[3], zfsDurationSeconds(m[4]) + scan.Errors, _ = strconv.Atoi(m[5]) + scan.Ended = zfsTime(m[6]) + case zfsScanRunning.MatchString(first): + m := zfsScanRunning.FindStringSubmatch(first) + scan.Kind, scan.State, scan.Started = m[1], "running", zfsTime(m[2]) + if p := zfsScanPercent.FindStringSubmatch(rest); p != nil { + scan.Percent, _ = strconv.ParseFloat(p[1], 64) + } + if g := zfsScanToGo.FindStringSubmatch(rest); g != nil { + scan.Remaining = zfsDurationSeconds(g[1]) + } + case zfsScanStopped.MatchString(first): + m := zfsScanStopped.FindStringSubmatch(first) + scan.Kind, scan.State = m[1], m[2] + if m[2] == "canceled" { + scan.Ended = zfsTime(m[3]) + } else { + scan.Started = zfsTime(m[3]) + } + default: + scan.Kind = "other" + } + return scan +} + +var zfsVdevName = regexp.MustCompile(`^(mirror|raidz[123]?|draid[123]?(?::\S+)?|replacing|spare)-\d+$`) + +func zfsIsState(s string) bool { + switch s { + case "ONLINE", "DEGRADED", "FAULTED", "OFFLINE", "UNAVAIL", "REMOVED", "SUSPENDED", "AVAIL", "INUSE": + return true + } + return false +} + +// parseZfsVdevs reads the rows of the config: section. The first row is the pool itself and is not returned (its children are); the other rows at the same +// depth (logs, cache, spares...) are groups. +func parseZfsVdevs(rows []string) []ZFSVdev { + type node struct { + vdev ZFSVdev + children []*node + } + var tops []*node + var stack []*node // stack[i] is the last row seen at depth i + var root *node + for _, row := range rows { + row = strings.TrimLeft(row, "\t") + trimmed := strings.TrimLeft(row, " ") + fields := strings.Fields(trimmed) + if len(fields) == 0 || (fields[0] == "NAME" && len(fields) > 1 && fields[1] == "STATE") { + continue + } + depth := (len(row) - len(trimmed)) / 2 + v := ZFSVdev{Name: fields[0], Read: -1, Write: -1, Cksum: -1} + if len(fields) >= 2 && zfsIsState(fields[1]) { + v.State = fields[1] + if len(fields) >= 5 { + v.Read, v.Write, v.Cksum = zfsInt(fields[2]), zfsInt(fields[3]), zfsInt(fields[4]) + v.Note = strings.Join(fields[5:], " ") + } + } + m := zfsVdevName.FindStringSubmatch(v.Name) + switch { + case m != nil: + v.Kind = strings.TrimRight(m[1], "0123") + if strings.HasPrefix(m[1], "draid") { + v.Kind = "draid" + } + case depth == 0 && root != nil: + v.Kind = "group" + default: + v.Kind = "disk" + if strings.HasPrefix(v.Name, "/dev/") { + v.Disk = zfsResolveDisk(v.Name) + } + } + n := &node{vdev: v} + switch { + case root == nil: + root = n + stack = []*node{n} // the pool: its children are the top of the tree + case depth == 0: + stack = []*node{n} + tops = append(tops, n) + default: + if depth > len(stack) { + depth = len(stack) + } + if parent := stack[depth-1]; parent == root { + tops = append(tops, n) + } else { + parent.children = append(parent.children, n) + } + stack = append(stack[:depth], n) + } + } + var build func(n *node) ZFSVdev + build = func(n *node) ZFSVdev { + v := n.vdev + for _, c := range n.children { + v.Children = append(v.Children, build(c)) + } + return v + } + out := []ZFSVdev{} + for _, n := range tops { + out = append(out, build(n)) + } + return out +} + +// zfsResolveDisk is the disk (sdb, nvme0n1) a device path is on, or empty, a var for the tests. +var zfsResolveDisk = func(path string) string { + real, err := filepath.EvalSymlinks(path) + if err != nil { + return "" + } + return diskOf("/sys", filepath.Base(real)) +} diff --git a/service/zfs_test.go b/service/zfs_test.go new file mode 100644 index 0000000..7235fb8 --- /dev/null +++ b/service/zfs_test.go @@ -0,0 +1,497 @@ +package service + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +func zfsFixture(t *testing.T, name string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "zfs", name)) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func zfsClearCache() { + zfsCache.Lock() + zfsCache.state, zfsCache.flight = nil, nil + zfsCache.Unlock() +} + +// zfsReset puts the package back as it was when the test ends. +func zfsReset(t *testing.T) { + t.Helper() + exec, has, resolve, now, grace := zfsExec, zfsHas, zfsResolveDisk, zfsNow, zfsKillGrace + zfsClearCache() + zfsResolveDisk = func(path string) string { return "nvme0n1" } + t.Cleanup(func() { + zfsExec, zfsHas, zfsResolveDisk, zfsNow, zfsKillGrace = exec, has, resolve, now, grace + zfsClearCache() + }) +} + +func parseStatus(t *testing.T, fixture string) ZFSPool { + t.Helper() + zfsReset(t) + pool := ZFSPool{Name: "p", DataErrors: -1} + parseZpoolStatus(zfsFixture(t, fixture), &pool) + return pool +} + +func TestParseZpoolList(t *testing.T) { + pools := parseZpoolList(zfsFixture(t, "zpool_list_two.txt")) + if len(pools) != 2 { + t.Fatalf("%+v", pools) + } + p := pools[0] + if p.Name != "pool1" || p.Size != 1992864825344 || p.Allocated != 1060842668032 || p.Free != 932022157312 || p.Fragmentation != 29 || p.Capacity != 53 || p.Health != "ONLINE" { + t.Fatalf("%+v", p) + } + if pools[1].Fragmentation != -1 || pools[1].Health != "DEGRADED" { + t.Fatalf("a '-' is not a number: %+v", pools[1]) + } + if got := parseZpoolList(zfsFixture(t, "zpool_list_none.txt")); len(got) != 0 { + t.Fatalf("'no pools available' is no pool: %+v", got) + } + if got := parseZpoolList(""); got == nil || len(got) != 0 { + t.Fatalf("%+v", got) + } +} + +func TestParseStatusHealthyMirror(t *testing.T) { + p := parseStatus(t, "zpool_status_healthy_mirror.txt") + if !p.Known || p.State != "ONLINE" || p.DataErrors != 0 || p.Status != "" { + t.Fatalf("%+v", p) + } + s := p.Scan + want := time.Date(2026, 10, 11, 0, 40, 48, 0, time.Local).Unix() + if s.Kind != "scrub" || s.State != "finished" || s.Repaired != "0B" || s.Errors != 0 || s.Duration != 16*60+47 || s.Ended != want { + t.Fatalf("%+v", s) + } + if len(p.Vdevs) != 1 { + t.Fatalf("the pool itself is not a vdev of the tree: %+v", p.Vdevs) + } + m := p.Vdevs[0] + if m.Name != "mirror-0" || m.Kind != "mirror" || m.State != "ONLINE" || len(m.Children) != 2 { + t.Fatalf("%+v", m) + } + d := m.Children[0] + if d.Kind != "disk" || d.Disk != "nvme0n1" || d.Read != 0 || d.Write != 0 || d.Cksum != 0 || !strings.HasSuffix(d.Name, "806391_1-part1") { + t.Fatalf("%+v", d) + } +} + +func TestParseStatusDegradedWithResilver(t *testing.T) { + p := parseStatus(t, "zpool_status_degraded_raidz.txt") + if !p.Known || p.State != "DEGRADED" || p.DataErrors != 2 { + t.Fatalf("%+v", p) + } + if !strings.HasPrefix(p.Status, "One or more devices could not be opened.") || !strings.Contains(p.Status, "degraded state.") || !strings.HasPrefix(p.Action, "Attach the missing device") || !strings.HasSuffix(p.See, "ZFS-8000-2Q") { + t.Fatalf("the words of ZFS are kept, joined: %q / %q / %q", p.Status, p.Action, p.See) + } + s := p.Scan + if s.Kind != "resilver" || s.State != "running" || s.Percent != 43.2 || s.Remaining != 20*60+11 || s.Started == 0 { + t.Fatalf("%+v", s) + } + if len(p.Vdevs) != 1 || p.Vdevs[0].Kind != "raidz" || p.Vdevs[0].State != "DEGRADED" || len(p.Vdevs[0].Children) != 3 { + t.Fatalf("%+v", p.Vdevs) + } + kids := p.Vdevs[0].Children + if kids[1].Write != 2 { + t.Fatalf("%+v", kids[1]) + } + if kids[2].State != "UNAVAIL" || kids[2].Note != "was /dev/sdc1" || kids[2].Disk != "" || kids[2].Name != "11485761428236503841" { + t.Fatalf("a disk that is gone is still in the tree: %+v", kids[2]) + } +} + +func TestParseStatusScrubRunningWithNoEstimate(t *testing.T) { + s := parseStatus(t, "zpool_status_scrub_running.txt").Scan + if s.Kind != "scrub" || s.State != "running" || s.Percent != 21.7 || s.Remaining != -1 { + t.Fatalf("%+v", s) + } +} + +func TestParseStatusGroups(t *testing.T) { + p := parseStatus(t, "zpool_status_groups.txt") + names := []string{} + for _, v := range p.Vdevs { + names = append(names, v.Name+":"+v.Kind) + } + if strings.Join(names, " ") != "mirror-0:mirror mirror-1:mirror logs:group cache:group spares:group" { + t.Fatalf("%v", names) + } + if len(p.Vdevs[0].Children) != 2 || len(p.Vdevs[2].Children) != 1 || p.Vdevs[2].Children[0].Name != "/dev/sde" { + t.Fatalf("%+v", p.Vdevs) + } + if spare := p.Vdevs[4].Children[0]; spare.State != "AVAIL" || spare.Read != -1 { + t.Fatalf("a spare has no counters: %+v", spare) + } + if p.Scan.Kind != "scrub" || p.Scan.State != "canceled" || p.Scan.Ended == 0 { + t.Fatalf("%+v", p.Scan) + } +} + +func TestParseStatusKeepsGoingOverWhatItDoesNotKnow(t *testing.T) { + zfsReset(t) + text := zfsFixture(t, "zpool_status_healthy_mirror.txt") + text = strings.Replace(text, "config:", "a line from a newer version of ZFS\nconfig:", 1) + text = strings.Replace(text, "errors:", "\tsome new thing\nerrors:", 1) + pool := ZFSPool{DataErrors: -1} + parseZpoolStatus(text, &pool) + if !pool.Known || pool.State != "ONLINE" || len(pool.Vdevs) != 1 || pool.DataErrors != 0 { + t.Fatalf("%+v", pool) + } +} + +func TestParseStatusTruncatedOrEmpty(t *testing.T) { + zfsReset(t) + pool := ZFSPool{DataErrors: -1} + parseZpoolStatus(" pool: tank\n state: ONLINE\n", &pool) + if !pool.Known || pool.State != "ONLINE" || len(pool.Vdevs) != 0 || pool.DataErrors != -1 || pool.Scan.Kind != "none" { + t.Fatalf("%+v", pool) + } + pool = ZFSPool{DataErrors: -1} + parseZpoolStatus("", &pool) + if pool.Known { + t.Fatalf("nothing was read, the pool is not known: %+v", pool) + } + pool = ZFSPool{DataErrors: -1} + parseZpoolStatus("cannot open 'x': no such pool\n", &pool) + if pool.Known { + t.Fatalf("%+v", pool) + } +} + +func TestParseScanNoneRequested(t *testing.T) { + pool := ZFSPool{} + parseZpoolStatus(" pool: t\n state: ONLINE\n scan: none requested\nconfig:\n\n", &pool) + if pool.Scan.Kind != "none" || pool.Scan.Raw != "" { + t.Fatalf("%+v", pool.Scan) + } +} + +func TestParseScanElsewhere(t *testing.T) { + scan := parseZfsScan([]string{"something new happened"}) + if scan.Kind != "other" || scan.Raw != "something new happened" { + t.Fatalf("%+v", scan) + } + if got := zfsDurationSeconds("1 days 02:03:04"); got != 86400+2*3600+3*60+4 { + t.Fatalf("%d", got) + } +} + +func TestParseZfsList(t *testing.T) { + datasets := parseZfsList(zfsFixture(t, "zfs_list_datasets.txt")) + if len(datasets) != 6 { + t.Fatalf("%+v", datasets) + } + root, app, vol, secret := datasets[0], datasets[2], datasets[4], datasets[5] + if root.Name != "pool1" || root.Pool != "pool1" || root.Type != "filesystem" || root.Mountpoint != "/pool1" || !root.Mounted || root.Quota != 0 || root.KeyStatus != "" { + t.Fatalf("%+v", root) + } + if app.Pool != "pool1" || app.Used != 300647710720 || app.UsedBySnapshots != 31457280 || app.Referenced != 300647710720 || app.Available != 671000000000 { + t.Fatalf("%+v", app) + } + if vol.Type != "volume" || vol.Mountpoint != "" || vol.Mounted { + t.Fatalf("a volume has no mount point: %+v", vol) + } + if secret.KeyStatus != "unavailable" || secret.Mounted || secret.Quota != 1073741824 { + t.Fatalf("%+v", secret) + } + if got := parseZfsList(zfsFixture(t, "zfs_list_root_only.txt")); len(got) != 1 || got[0].Name != "pool1" { + t.Fatalf("a pool with no datasets of its own: %+v", got) + } + if got := parseZfsList(zfsFixture(t, "zfs_list_none.txt")); got == nil || len(got) != 0 { + t.Fatalf("'no datasets available' is no dataset: %+v", got) + } +} + +func TestParseZfsSnapshots(t *testing.T) { + list := parseZfsSnapshots(zfsFixture(t, "zfs_list_snapshots.txt")) + if len(list) != 2 || list[0].Name != "backup_20261105_023008" || list[0].Used != 12373196 || list[0].Written != 17000000 || list[0].Created != 1762309808 || list[1].Written != 299000000000 { + t.Fatalf("%+v", list) + } + if got := parseZfsSnapshots("not a snapshot\t1\t2\t3\t4\n"); len(got) != 0 { + t.Fatalf("%+v", got) + } +} + +func TestParseKstatIO(t *testing.T) { + io, ok := parseZfsKstatIO(zfsFixture(t, "kstat_io.txt")) + if !ok || io.ReadBytes != 9990000000 || io.WriteBytes != 7020000000 || io.ReadOps != 1320000 || io.WriteOps != 1740000 { + t.Fatalf("%v %+v", ok, io) + } + for _, bad := range []string{"", "1 2 3\n", "nread nwritten\n1 2\n", "nread nwritten reads writes\nx y z w\n"} { + if _, ok := parseZfsKstatIO(bad); ok { + t.Fatalf("%q was taken for good", bad) + } + } +} + +func TestZFSIOReadsEveryPool(t *testing.T) { + zfsReset(t) + proc := t.TempDir() + for _, pool := range []string{"tank", "pool1"} { + if err := os.MkdirAll(filepath.Join(proc, pool), 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(proc, "pool1", "io"), []byte(zfsFixture(t, "kstat_io.txt")), 0o644); err != nil { + t.Fatal(err) + } + zfsNow = func() time.Time { return time.UnixMilli(1762309808123) } + got := ZFSIO(proc) + if len(got) != 1 || got[0].Pool != "pool1" || got[0].At != 1762309808123 || got[0].ReadOps != 1320000 { + t.Fatalf("a pool with no counters is left out, not shown at zero: %+v", got) + } + if got := ZFSIO(filepath.Join(proc, "nowhere")); got == nil || len(got) != 0 { + t.Fatalf("%+v", got) + } +} + +// fakeZFS answers the commands with the fixtures, keyed by " ", and fails the ones in fail. A snapshot listing is keyed "zfs snapshot". +type fakeZFS struct { + calls atomic.Int32 + seen []string + out map[string]string + fail map[string]error +} + +func healthyFake(t *testing.T) *fakeZFS { + t.Helper() + zfsReset(t) + f := &fakeZFS{ + out: map[string]string{ + "zpool list": zfsFixture(t, "zpool_list_one.txt"), + "zpool status": zfsFixture(t, "zpool_status_healthy_mirror.txt"), + "zfs list": zfsFixture(t, "zfs_list_datasets.txt"), + "zfs snapshot": zfsFixture(t, "zfs_list_snapshots.txt"), + }, + fail: map[string]error{}, + } + zfsHas = func() bool { return true } + zfsExec = func(ctx context.Context, name string, args ...string) ([]byte, error) { + f.calls.Add(1) + f.seen = append(f.seen, name+" "+strings.Join(args, " ")) + key := name + " " + args[0] + if name == "zfs" && strings.Contains(strings.Join(args, " "), "-t snapshot") { + key = "zfs snapshot" + } + if err := f.fail[key]; err != nil { + return nil, err + } + return []byte(f.out[key]), nil + } + return f +} + +func TestZFSOverview(t *testing.T) { + f := healthyFake(t) + state := ZFSOverview() + if !state.Available || state.Busy || state.Stale || len(state.Pools) != 1 || !state.Pools[0].Known || state.Pools[0].State != "ONLINE" || !state.DatasetsKnown || len(state.Datasets) != 6 { + t.Fatalf("%+v", state) + } + // only these three commands, in the forms that were agreed + want := []string{ + "zpool list -H -p -o name,size,allocated,free,fragmentation,capacity,health", + "zpool status -p -P pool1", + "zfs list -H -p -t filesystem,volume -o name,type,used,available,referenced,usedbysnapshots,mountpoint,mounted,quota,keystatus", + } + if strings.Join(f.seen, "\n") != strings.Join(want, "\n") { + t.Fatalf("%q", f.seen) + } + ZFSOverview() + if f.calls.Load() != 3 { + t.Fatalf("a second look within the TTL asks nothing more: %d commands", f.calls.Load()) + } +} + +func TestZFSOverviewAgainAfterTheTTL(t *testing.T) { + f := healthyFake(t) + now := time.Now() + zfsNow = func() time.Time { return now } + ZFSOverview() + now = now.Add(zfsListTTL + time.Second) + ZFSOverview() + if f.calls.Load() != 6 { + t.Fatalf("%d", f.calls.Load()) + } +} + +func TestZFSNotThere(t *testing.T) { + f := healthyFake(t) + zfsHas = func() bool { return false } + state := ZFSOverview() + if state.Available || state.Reason != "tools_missing" || f.calls.Load() != 0 || state.Pools == nil || state.Datasets == nil { + t.Fatalf("%+v", state) + } +} + +func TestZFSModuleNotLoaded(t *testing.T) { + f := healthyFake(t) + f.fail["zpool list"] = errors.New("zpool: exit status 1: The ZFS modules are not loaded.\nTry running '/sbin/modprobe zfs' as root to load them.") + state := ZFSOverview() + if state.Available || state.Reason != "module_missing" { + t.Fatalf("%+v", state) + } +} + +func TestZFSNoPool(t *testing.T) { + f := healthyFake(t) + f.out["zpool list"] = zfsFixture(t, "zpool_list_none.txt") + state := ZFSOverview() + if !state.Available || len(state.Pools) != 0 || f.calls.Load() != 1 { + t.Fatalf("%+v after %d commands", state, f.calls.Load()) + } +} + +func TestZFSPoolWithNoDatasetsOfItsOwn(t *testing.T) { + f := healthyFake(t) + f.out["zfs list"] = zfsFixture(t, "zfs_list_root_only.txt") + state := ZFSOverview() + if !state.DatasetsKnown || len(state.Datasets) != 1 { + t.Fatalf("%+v", state) + } + zfsClearCache() + f.out["zfs list"] = zfsFixture(t, "zfs_list_none.txt") + state = ZFSOverview() + if !state.DatasetsKnown || len(state.Datasets) != 0 { + t.Fatalf("%+v", state) + } +} + +func TestZFSDegradedPoolIsTold(t *testing.T) { + f := healthyFake(t) + f.out["zpool list"] = zfsFixture(t, "zpool_list_two.txt") + f.out["zpool status"] = zfsFixture(t, "zpool_status_degraded_raidz.txt") + state := ZFSOverview() + if len(state.Pools) != 2 || state.Pools[1].State != "DEGRADED" || !state.Pools[1].Known { + t.Fatalf("%+v", state.Pools) + } +} + +func TestZFSUnreadableStatusIsNotHealthy(t *testing.T) { + f := healthyFake(t) + f.fail["zpool status"] = errors.New("zpool: signal: killed") + p := ZFSOverview().Pools[0] + if p.Known || p.State != "" || p.Size != 1992864825344 || p.Health != "ONLINE" { + t.Fatalf("the figures stay, the state is not claimed: %+v", p) + } + zfsClearCache() + f.fail["zfs list"] = errors.New("zfs: timeout") + state := ZFSOverview() + if state.DatasetsKnown || len(state.Datasets) != 0 { + t.Fatalf("%+v", state) + } +} + +func TestZFSFailureKeepsTheLastAnswerMarkedStale(t *testing.T) { + f := healthyFake(t) + now := time.Now() + zfsNow = func() time.Time { return now } + ZFSOverview() + f.fail["zpool list"] = errors.New("zpool: exit status 1: something broke") + now = now.Add(zfsListTTL + time.Second) + state := ZFSOverview() + if !state.Available || len(state.Pools) != 1 || !state.Stale { + t.Fatalf("%+v", state) + } + // with nothing to fall back on it is busy, which the dashboard shows as "not yet", not as "no ZFS" + zfsClearCache() + state = ZFSOverview() + if !state.Available || !state.Busy { + t.Fatalf("%+v", state) + } +} + +func TestZFSSnapshotsOnlyOfDatasetsTheBoxLists(t *testing.T) { + f := healthyFake(t) + for _, name := range []string{"nope", "pool1/zzz", "-r", "pool1/appdata; reboot", ""} { + if _, err := ZFSSnapshotsOf(name); !errors.Is(err, ErrZFSNoDataset) { + t.Fatalf("%q: %v", name, err) + } + } + if f.calls.Load() != 3 { + t.Fatalf("the overview only, and no snapshot command for a name that is not listed: %d commands", f.calls.Load()) + } +} + +func TestZFSSnapshotsOfADataset(t *testing.T) { + f := healthyFake(t) + list, err := ZFSSnapshotsOf("pool1/appdata") + if err != nil || list.Dataset != "pool1/appdata" || len(list.Snapshots) != 2 || list.Truncated { + t.Fatalf("%+v %v", list, err) + } + if last := f.seen[len(f.seen)-1]; last != "zfs list -H -p -t snapshot -d 1 -S creation -o name,used,referenced,written,creation pool1/appdata" { + t.Fatalf("%q", last) + } +} + +func TestZFSSnapshotsAreCutAtTheLimit(t *testing.T) { + f := healthyFake(t) + f.out["zfs snapshot"] = strings.Repeat("pool1/appdata@s\t1\t2\t3\t4\n", ZFSSnapshotLimit+50) + list, err := ZFSSnapshotsOf("pool1/appdata") + if err != nil || len(list.Snapshots) != ZFSSnapshotLimit || !list.Truncated { + t.Fatalf("%d %v %v", len(list.Snapshots), list.Truncated, err) + } +} + +func TestZFSSnapshotsWithoutZFS(t *testing.T) { + healthyFake(t) + zfsHas = func() bool { return false } + if _, err := ZFSSnapshotsOf("pool1"); !errors.Is(err, ErrZFSUnavailable) { + t.Fatal(err) + } +} + +// A command that does not come back must not be followed by another one. +func TestZFSStuckCommandIsNotPiledUpOn(t *testing.T) { + zfsReset(t) + zfsKillGrace = 20 * time.Millisecond + release := make(chan struct{}) + var started atomic.Int32 + zfsExec = func(ctx context.Context, name string, args ...string) ([]byte, error) { + started.Add(1) + <-release // ignores the context, as a process stuck in the kernel does + return nil, nil + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := runZFS(ctx, 30*time.Millisecond, "zpool", "status"); err == nil { + t.Fatal("a command that never came back was taken for done") + } + short, cancelShort := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancelShort() + if _, err := runZFS(short, time.Second, "zpool", "status"); !errors.Is(err, ErrZFSBusy) { + t.Fatalf("%v", err) + } + if started.Load() != 1 { + t.Fatalf("%d processes", started.Load()) + } + close(release) + deadline := time.Now().Add(2 * time.Second) + for len(zfsSlot) != 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if len(zfsSlot) != 0 { + t.Fatal("the slot was not given back when the command ended") + } +} + +func TestCappedBufferKeepsTheStart(t *testing.T) { + b := &cappedBuffer{max: 5} + n, _ := b.Write([]byte("abc")) + m, _ := b.Write([]byte("defgh")) + if n != 3 || m != 5 || b.String() != "abcde" { + t.Fatalf("%d %d %q", n, m, b.String()) + } +} From ec2498103164e679cd944d44d10d07be73f564e1 Mon Sep 17 00:00:00 2001 From: Gary Date: Mon, 9 Nov 2026 12:10:19 +0100 Subject: [PATCH 2/2] test(zfs): check the parsers on the output of a real OpenZFS 2.4.1 box zpool list, zfs list and zpool status of a healthy mirror, pasted by a user. --- service/testdata/zfs/README.md | 11 +++++---- service/testdata/zfs/real_zfs_list.txt | 4 ++++ service/testdata/zfs/real_zpool_list.txt | 1 + .../zfs/real_zpool_status_features.txt | 17 ++++++++++++++ service/zfs_test.go | 23 +++++++++++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 service/testdata/zfs/real_zfs_list.txt create mode 100644 service/testdata/zfs/real_zpool_list.txt create mode 100644 service/testdata/zfs/real_zpool_status_features.txt diff --git a/service/testdata/zfs/README.md b/service/testdata/zfs/README.md index 2c8af67..f3c041d 100644 --- a/service/testdata/zfs/README.md +++ b/service/testdata/zfs/README.md @@ -2,7 +2,10 @@ The outputs of `zpool` and `zfs` the parsers in `service/zfs.go` are tested on. -**These files were written from the OpenZFS documentation and man pages, and from the output a user pasted in -ReCasaOS/CasaOS#66 (the healthy mirror), not captured from a box.** Replace them with real captures when there are some -(`zpool list -H -p -o ...`, `zpool status -p -P `, `zfs list -H -p ...`, on OpenZFS 2.1, 2.2 and 2.3), and take this -note out then. +- `real_*`: **captured by a user on a real box** (se3, ReCasaOS/CasaOS discussion 66, 2026-11-09: OpenZFS 2.4.1, Ubuntu 26.04): + `zpool list -H -p`, `zfs list -H -p` and the text of `zpool status -p -P` of a healthy mirror whose pool does not have all + features enabled. In the pasted `zpool status` the indentation of the tree was lost by the forum, so `real_zpool_status_features.txt` + has the words and numbers as pasted with the tab and two-space indentation `zpool` prints put back. +- The other files are **written from the OpenZFS documentation and man pages**, not captured: a degraded raidz, scrubs and + resilvers in progress or canceled, logs/cache/spares, a volume, a locked dataset, no pool, no dataset, kstat counters. + Replace them with real captures when there are some (OpenZFS 2.1, 2.2, 2.3), and take this note out then. diff --git a/service/testdata/zfs/real_zfs_list.txt b/service/testdata/zfs/real_zfs_list.txt new file mode 100644 index 0000000..1b0d7d7 --- /dev/null +++ b/service/testdata/zfs/real_zfs_list.txt @@ -0,0 +1,4 @@ +pool1 filesystem 1523315021414 607096213504 106496 65536 /pool1 yes 0 - +pool1/appdata filesystem 303757225984 407809804698 302870736896 886489088 /pool1/data1 yes 0 - +pool1/bitcoin-data filesystem 1018895228928 407809804698 1018895228928 0 /pool1/bitcoin-data yes 0 - +pool1/immich-db filesystem 1063133184 407809804698 1056239616 6893568 /pool1/immich-db yes 0 - diff --git a/service/testdata/zfs/real_zpool_list.txt b/service/testdata/zfs/real_zpool_list.txt new file mode 100644 index 0000000..bce5b9b --- /dev/null +++ b/service/testdata/zfs/real_zpool_list.txt @@ -0,0 +1 @@ +pool1 1992864825344 1324916645888 667948179456 64 66 ONLINE diff --git a/service/testdata/zfs/real_zpool_status_features.txt b/service/testdata/zfs/real_zpool_status_features.txt new file mode 100644 index 0000000..b0f223a --- /dev/null +++ b/service/testdata/zfs/real_zpool_status_features.txt @@ -0,0 +1,17 @@ + pool: pool1 + state: ONLINE +status: Some supported and requested features are not enabled on the pool. + The pool can still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(7) for details. + scan: scrub repaired 0B in 01:21:07 with 0 errors on Sun Nov 8 01:45:08 2026 +config: + + NAME STATE READ WRITE CKSUM + pool1 ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + /dev/disk/by-id/nvme-WD_BLACK_SN770_2TB_24443A806391_1-part1 ONLINE 0 0 0 + /dev/disk/by-id/nvme-WD_BLACK_SN770_2TB_24443A805750_1-part1 ONLINE 0 0 0 + +errors: No known data errors diff --git a/service/zfs_test.go b/service/zfs_test.go index 7235fb8..fbc3ed8 100644 --- a/service/zfs_test.go +++ b/service/zfs_test.go @@ -487,6 +487,29 @@ func TestZFSStuckCommandIsNotPiledUpOn(t *testing.T) { } } +// What a real box (OpenZFS 2.4.1, Ubuntu 26.04) printed. +func TestRealOutputOfOpenZFS24(t *testing.T) { + pools := parseZpoolList(zfsFixture(t, "real_zpool_list.txt")) + if len(pools) != 1 || pools[0].Size != 1992864825344 || pools[0].Allocated != 1324916645888 || pools[0].Free != 667948179456 || pools[0].Fragmentation != 64 || pools[0].Capacity != 66 || pools[0].Health != "ONLINE" { + t.Fatalf("%+v", pools) + } + datasets := parseZfsList(zfsFixture(t, "real_zfs_list.txt")) + if len(datasets) != 4 || datasets[1].Name != "pool1/appdata" || datasets[1].Mountpoint != "/pool1/data1" || datasets[1].UsedBySnapshots != 886489088 || datasets[0].Referenced != 106496 { + t.Fatalf("a mount point is not the name of the dataset: %+v", datasets) + } + p := parseStatus(t, "real_zpool_status_features.txt") + // the pool is healthy and says something: features not enabled is a note, not a state + if !p.Known || p.State != "ONLINE" || !strings.HasPrefix(p.Status, "Some supported and requested features are not enabled") || !strings.Contains(p.Action, "zpool upgrade") || !strings.HasSuffix(p.Action, "for details.") { + t.Fatalf("%q / %q / %q", p.State, p.Status, p.Action) + } + if p.Scan.Kind != "scrub" || p.Scan.Duration != 1*3600+21*60+7 || p.Scan.Errors != 0 || p.Scan.Ended != time.Date(2026, 11, 8, 1, 45, 8, 0, time.Local).Unix() { + t.Fatalf("%+v", p.Scan) + } + if len(p.Vdevs) != 1 || p.Vdevs[0].Name != "mirror-0" || len(p.Vdevs[0].Children) != 2 || p.DataErrors != 0 { + t.Fatalf("%+v", p.Vdevs) + } +} + func TestCappedBufferKeepsTheStart(t *testing.T) { b := &cappedBuffer{max: 5} n, _ := b.Write([]byte("abc"))