diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 8aa70ac..9852e7b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -56,6 +56,9 @@ jobs: - name: Check the fetch item with and without the old batch file, and that it keeps LOCALAPPDATA private shell: pwsh run: ./tests/fetch-item.ps1 + - name: Check the dataverify item pattern, plan and mismatch reporting against a fake guest + shell: pwsh + run: ./tests/dataverify.ps1 - name: Scan tracked files for private identity and paths run: python3 .modavd/scripts/privacy.py . diff --git a/README.en.md b/README.en.md index b0ceaa5..541c7d5 100644 --- a/README.en.md +++ b/README.en.md @@ -27,7 +27,7 @@ and the bundled scripts in `package/` and the bridge in `bridge/` of the ModAVD ## How to run -PowerShell 7 (`pwsh`); the helper scripts use Python 3.12. +PowerShell 7 (`pwsh`); the helper scripts use Python 3.12; the `dataverify` item also needs Go on the host (it builds the small writer that runs in the guest). ```powershell pwsh -NoProfile -File regress.ps1 -Suite suites\release.psd1 -Scope emulator -Sdk -Baseline @@ -69,7 +69,7 @@ The suite refers to keys with `{Local:}`, and the item scripts take their d | `PackageScripts` | The `package\` of the ModAVD repository (the bridge install script `modavd-bridge.ps1` is in it) | runner, bridge, upgrade, game-real | | `ProductionRoots` | Root folders the tests never touch, in addition to `D:\ModAVD` (not instead of it): list the root you use yourself every day here too. A test root, output, work or instance folder under them stops the run | runner, all items | | `TestRoots` | The only test roots the regression may use (optional): when set, only the listed root folders themselves are accepted, and a `TestRoot`, `PackageRoot` or `-Root` pointing elsewhere stops the run | runner, all items | -| `TestRoot` | The ModAVD root folder used for testing: the existing flow-a / flow-b under `E:\testbed`; a folder that is not an existing root (no `mas-avd.json`), or whose `mas-avd.json` does not set its own `adbServerPort` and `scriptAdbServerPort`, stops the run, none is created (`portBase` when the machine also has a production root folder; instance 0 has the bridge installed and is logged in with the *Onmyoji* test account; the other numbers are created temporarily and deleted after the run) | bridge, bridge-default, datadisk, yys | +| `TestRoot` | The ModAVD root folder used for testing: the existing flow-a / flow-b under `E:\testbed`; a folder that is not an existing root (no `mas-avd.json`), or whose `mas-avd.json` does not set its own `adbServerPort` and `scriptAdbServerPort`, stops the run, none is created (`portBase` when the machine also has a production root folder; instance 0 has the bridge installed and is logged in with the *Onmyoji* test account; the other numbers are created temporarily and deleted after the run) | bridge, bridge-default, datadisk, dataverify, yys | | `PackageRoot` | The test root for the items that install a package (`TestRoot` when not set): it is turned into the package under test first and stays on the new package afterwards | upgrade, ime-pkg | | `PrevPackage` | The package staging folder of the previous production version (a fixed copy) | upgrade | | `Launcher` | The launcher `avd-p0.ps1` (starts an AVD headless or windowed, with parameters such as `-AdbPort` and `-VulkanFix`) | game-real, ime | @@ -130,6 +130,7 @@ In the game UI, do only fixed operations such as "launch" and "tap Enter Game". | ark / r1999 / sr / ef | `items\game-real.ps1` | A 15-minute real run of each game with its own automation scripts (the round with the bridge installed), with the frame rate compared with `baselines\fps.json` | | bridge | `items\bridge.ps1` | The whole flow of installing / uninstalling the bridge (Windows PowerShell 5.1); it creates instance 6 and deletes it after the run | | datadisk | `items\datadisk.ps1` | The inode cap on a new disk and growing with `set --data-gb`; it creates instance 3 and deletes it after the run | +| dataverify | `items\dataverify.ps1` | Disk read/write data check (ModAVD/ModAVD#138): creates instance 4 and, under `/data/unencrypted/` in the guest, writes a 1 GiB file whose pattern is computed from a seed (single writes of 4 KiB–1 MiB, ordinary and `O_DIRECT` writes mixed), then runs `sync`, drops the page cache and reads the hash back; after a normal shutdown and a cold boot it reads the hash again and compares it with the one the host computes from the seed, and on a mismatch pulls the file back and reports the first differing offset and the number of bad bytes; the file and the instance are deleted afterwards. `-SizeMiB` changes the total, `-Seed` and `-DirectPercent` the pattern and the share of direct writes, `-Keep` keeps the instance. The guest `dd` has no `oflag=direct`, so the writing is done by `items\dataverify-guest.go`, which needs Go on the host to build | | upgrade | `items\upgrade.ps1` | Installs the previous release package into `PackageRoot` following the package manifests, creates a temporary instance and writes data, overlays the new package following "How to update", then checks boot, data, growing the disk, and that a normal boot installs the bridge automatically. When `-Next` is a package staging folder (with `components\modavd-ime.apk`), it also checks that the host IME came with the package, that the installed one is the officially signed one, and that it is set as the default (not checked when `-Next` is an SDK). Afterwards the temporary instance is deleted and the root folder stays on the new package | | yys | `items\yys.ps1` | *Onmyoji* cold-boots into the game and the frame rate is measured (instance 0 of the test root, bridge installed, already logged in); then MaaYYs (`items\maayys_run.py`) really fights Mitama battles | | ef-ab | `items\ef-ab.ps1` | *Arknights: Endfield* cold boots, alternating new and old builds; it measures only when the host is idle | diff --git a/README.md b/README.md index 91db49a..e9830e8 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ ModAVD 仓库里的随包脚本 `package/` 和执行桥 `bridge/`,所以单独 ## 怎么跑 -PowerShell 7(`pwsh`),辅助脚本用 Python 3.12。 +PowerShell 7(`pwsh`),辅助脚本用 Python 3.12;`dataverify` 项还要宿主装有 Go(编客体里的写入小程序)。 ```powershell pwsh -NoProfile -File regress.ps1 -Suite suites\release.psd1 -Scope emulator -Sdk <新 SDK> -Baseline <上一个发布版的 SDK> @@ -69,7 +69,7 @@ pwsh -NoProfile -File regress.ps1 -Suite suites\release.psd1 -Scope emulator -Sd | `PackageScripts` | ModAVD 仓库的 `package\`(装桥脚本 `modavd-bridge.ps1` 在里面) | 运行器、bridge、upgrade、game-real | | `ProductionRoots` | 测试不碰的根目录,在 `D:\ModAVD` 之外再加(不是替换它):你自己日常在用的根也写进来。测试根、输出、工作目录、实例目录在它们下面就停 | 运行器、各项 | | `TestRoots` | 回归只许用的测试根(可以不写):写了就只认列出来的根目录本身,`TestRoot`、`PackageRoot`、`-Root` 指到别处就停 | 运行器、各项 | -| `TestRoot` | 测试用的 ModAVD 根目录:`E:\testbed` 下已有的 flow-a / flow-b(`mas-avd.json` 里要写自己的 `adbServerPort`、`scriptAdbServerPort` 和 `portBase`;0 号已装桥、登着阴阳师测试号,其余编号临时建、跑完删)。不是已有根目录(没有 `mas-avd.json`)、没写这两个 adb 端口就停,不新建 | bridge、bridge-default、datadisk、yys | +| `TestRoot` | 测试用的 ModAVD 根目录:`E:\testbed` 下已有的 flow-a / flow-b(`mas-avd.json` 里要写自己的 `adbServerPort`、`scriptAdbServerPort` 和 `portBase`;0 号已装桥、登着阴阳师测试号,其余编号临时建、跑完删)。不是已有根目录(没有 `mas-avd.json`)、没写这两个 adb 端口就停,不新建 | bridge、bridge-default、datadisk、dataverify、yys | | `PackageRoot` | 装包类的项用的测试根(不写就用 `TestRoot`):先装成要测的包,测完根目录停在新包上 | upgrade、ime-pkg | | `PrevPackage` | 上一个生产版的组包目录(固定副本) | upgrade | | `ComponentCache` | 放着 `platform-tools_r37.0.1-win.zip`、`x86_64-34_r14.zip` 的目录(可以没有:没有就从网上下载一次) | fetch | @@ -131,6 +131,7 @@ REGRESS-RESULT {"pass":true,"summary":"到登录页 41 秒;进游戏后 52.7 f | ark / r1999 / sr / ef | `items\game-real.ps1` | 各游戏自己的自动化脚本实跑 15 分钟(装桥那轮),帧率和 `baselines\fps.json` 比 | | bridge | `items\bridge.ps1` | 执行桥装 / 卸全流程(Windows PowerShell 5.1),新建 6 号实例跑完删掉 | | datadisk | `items\datadisk.ps1` | 新盘 inode 封顶、`set --data-gb` 扩容,新建 3 号实例跑完删掉 | +| dataverify | `items\dataverify.ps1` | 磁盘读写数据核对(ModAVD/ModAVD#138):新建 4 号实例,在客体 `/data/unencrypted/` 里写一个由种子算出的 1 GiB 花样文件(单次 4 KiB–1 MiB,普通写和 `O_DIRECT` 混着),`sync`、清页缓存回读一次哈希;再正常关机、冷启动,回读哈希和宿主按种子算的比对,不一致就拉回来报第一个不同的偏移和错字节数;跑完删文件、删实例。`-SizeMiB` 改总量,`-Seed`、`-DirectPercent` 改花样和直写比例,`-Keep` 留着实例。客体的 `dd` 没有 `oflag=direct`,写入用 `items\dataverify-guest.go`,宿主上要有 Go 来编 | | upgrade | `items\upgrade.ps1` | 在 `PackageRoot` 上先按包根清单装回上一版正式包、建临时实例写数据,照「怎么更新」覆盖新包后核对开机、数据、扩容、照常开机自动装桥;`-Next` 是组包目录(带 `components\modavd-ime.apk`)时还核对宿主输入法跟着包过来、装上的是正式签名那份并设为默认(`-Next` 是 SDK 时不查);跑完删临时实例,根目录停在新包上 | | yys | `items\yys.ps1` | 阴阳师冷启动进游戏测帧(测试根的 0 号,已装桥、已登录),再用 MaaYYs(`items\maayys_run.py`)真打御魂 | | ef-ab | `items\ef-ab.ps1` | 终末地冷启动新旧交替,等宿主空闲才测 | diff --git a/items/dataverify-guest.go b/items/dataverify-guest.go new file mode 100644 index 0000000..8725c60 --- /dev/null +++ b/items/dataverify-guest.go @@ -0,0 +1,80 @@ +// dataverify 项在客体(Android,linux/amd64)里跑的小程序:按写入计划把花样文件的一段段字节写进数据文件, +// 计划里标了直写的块用 O_DIRECT 写,其余用普通写。客体自带的 toybox dd 没有 oflag=direct,所以需要它。 +// 不读不写别的东西:花样、计划都由宿主(items\dataverify-lib.ps1)算好推进来,写完的哈希由客体的 sha256sum 算。 +// 宿主上的 items\dataverify.ps1 用 `go build`(GOOS=linux GOARCH=amd64 CGO_ENABLED=0)编它,不要在 Windows 宿主上运行。 +// +// dvwrite <花样文件> <数据文件> <计划文件> +// +// 计划文件每行 `<偏移> <大小> <0|1>`(1 = 直写);文件偏移 o 处写的是花样[o mod 花样长度] 起的 <大小> 字节。 +package main + +import ( + "bufio" + "fmt" + "os" + "syscall" +) + +func fail(format string, a ...any) { + fmt.Printf("DV-WRITE-FAIL "+format+"\n", a...) + os.Exit(1) +} + +func main() { + if len(os.Args) != 4 { + fmt.Fprintln(os.Stderr, "usage: dvwrite ") + os.Exit(2) + } + pattern, err := os.ReadFile(os.Args[1]) + if err != nil || len(pattern) == 0 { + fail("read pattern: %v", err) + } + bf, err := os.OpenFile(os.Args[2], os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + fail("open buffered: %v", err) + } + df, err := os.OpenFile(os.Args[2], os.O_WRONLY|syscall.O_DIRECT, 0) + if err != nil { + fail("open O_DIRECT: %v", err) + } + // O_DIRECT 要内存对齐:匿名映射是页对齐的 + buf, err := syscall.Mmap(-1, 0, 1<<20, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE) + if err != nil { + fail("mmap: %v", err) + } + pf, err := os.Open(os.Args[3]) + if err != nil { + fail("open plan: %v", err) + } + sc := bufio.NewScanner(pf) + chunks, direct := 0, 0 + for sc.Scan() { + var off int64 + var size, d int + if _, err := fmt.Sscanf(sc.Text(), "%d %d %d", &off, &size, &d); err != nil { + fail("plan line %q: %v", sc.Text(), err) + } + start := int(off % int64(len(pattern))) + if size <= 0 || size > len(buf) || start+size > len(pattern) { + fail("chunk %d@%d out of range", size, off) + } + src := pattern[start : start+size] + f := bf + if d == 1 { + copy(buf, src) + src, f = buf[:size], df + direct++ + } + if n, err := f.WriteAt(src, off); err != nil || n != size { + fail("write %d@%d direct=%d: n=%d err=%v", size, off, d, n, err) + } + chunks++ + } + for _, f := range []*os.File{bf, df} { + if err := f.Sync(); err != nil { + fail("fsync: %v", err) + } + f.Close() + } + fmt.Printf("DV-WRITE-OK chunks=%d direct=%d\n", chunks, direct) +} diff --git a/items/dataverify-lib.ps1 b/items/dataverify-lib.ps1 new file mode 100644 index 0000000..6625f10 --- /dev/null +++ b/items/dataverify-lib.ps1 @@ -0,0 +1,189 @@ +# dataverify 项的主体(. "$PSScriptRoot\dataverify-lib.ps1"):花样、写入计划、客体计划文件、核对。不依赖 common.ps1, +# 所有碰客体 / 模拟器的动作由调用方以脚本块传进来(Invoke-DataVerify 的 $Ops),tests\dataverify.ps1 换上假的测它。 +# +# 花样:长 7 MiB 的"种子花样"(splitmix64 从种子算出,可重算)。文件偏移 o 处的字节 = 花样[o mod 7 MiB]。 +# 写入计划:文件按 1 MiB 分段,每段随机对半劈成 4 KiB–1 MiB 的 2 的幂大小的块(块的偏移是自己大小的整数倍), +# 每块随机用普通写或 O_DIRECT 写。这样每块都正好是花样里的一段连续字节,客体小程序一次 pwrite 写一块。计划和花样只由种子(和总量、直写比例)决定,宿主可以重算。 + +Add-Type -TypeDefinition @' +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; + +public static class DataVerifyNative +{ + public const int PatternLength = 7 << 20; + + static ulong Mix(ulong x) + { + unchecked + { + x += 0x9e3779b97f4a7c15UL; + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9UL; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebUL; + return x ^ (x >> 31); + } + } + + public static byte[] Pattern(ulong seed) + { + var p = new byte[PatternLength]; + ulong s = seed; + for (int i = 0; i < p.Length; i += 8) + { + s = Mix(s); + BitConverter.GetBytes(s).CopyTo(p, i); + } + return p; + } + + class Rng + { + ulong s; + public Rng(ulong seed) { s = seed; } + public ulong Next() { s = Mix(s); return s; } + } + + // 返回 (偏移, 大小, 是否直写) 三个一组的平铺数组 + public static long[] Plan(ulong seed, int mib, int directPercent) + { + var r = new Rng(seed ^ 0x77UL); + var list = new List(); + for (int m = 0; m < mib; m++) Split(r, list, (long)m << 20, 1 << 20, directPercent); + return list.ToArray(); + } + + static void Split(Rng r, List list, long off, int size, int directPercent) + { + if (size == 4096 || r.Next() % 100 < 40) + { + list.Add(off); list.Add(size); list.Add((long)(r.Next() % 100) < directPercent ? 1 : 0); + return; + } + Split(r, list, off, size / 2, directPercent); + Split(r, list, off + size / 2, size / 2, directPercent); + } + + // 文件前 total 字节应有的 SHA-256(小写十六进制) + public static string ExpectedSha256(byte[] pattern, long total) + { + using (var h = SHA256.Create()) + { + long o = 0; + while (o < total) + { + int start = (int)(o % pattern.Length); + int n = (int)Math.Min(pattern.Length - start, total - o); + h.TransformBlock(pattern, start, n, null, 0); + o += n; + } + h.TransformFinalBlock(new byte[0], 0, 0); + return BitConverter.ToString(h.Hash).Replace("-", "").ToLowerInvariant(); + } + } + + // 逐字节比对:返回 {第一个不同的偏移(没有是 -1), 错字节数, 文件实际大小}。比期望短的部分算全错,多出来的部分也算错。 + public static long[] Diff(byte[] pattern, string path, long total) + { + long first = -1, bad = 0, size; + using (var f = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20)) + { + size = f.Length; + var buf = new byte[1 << 20]; + long o = 0; + while (o < Math.Max(size, total)) + { + int k = o < size ? f.Read(buf, 0, buf.Length) : 0; + if (k == 0 && o < size) break; + int n = k > 0 ? k : (int)Math.Min(buf.Length, total - o); + for (int i = 0; i < n; i++) + { + long pos = o + i; + bool ok = k > 0 && pos < total && buf[i] == pattern[(int)(pos % pattern.Length)]; + if (!ok) { if (first < 0) first = pos; bad++; } + } + o += n; + } + } + return new long[] { first, bad, size }; + } +} +'@ + +function New-DataPattern([uint64]$Seed) { , [DataVerifyNative]::Pattern($Seed) } + +# 写入计划:[pscustomobject]@{ Offset; Size; Direct } 的数组 +function New-WritePlan([uint64]$Seed, [int]$SizeMiB, [int]$DirectPercent = 50) { + if ($SizeMiB -lt 1) { throw "总量要至少 1 MiB($SizeMiB)" } + $flat = [DataVerifyNative]::Plan($Seed, $SizeMiB, $DirectPercent) + for ($i = 0; $i -lt $flat.Length; $i += 3) { + [pscustomobject]@{ Offset = $flat[$i]; Size = [int]$flat[$i + 1]; Direct = ($flat[$i + 2] -eq 1) } + } +} + +function Get-ExpectedSha256([byte[]]$Pattern, [int64]$Bytes) { [DataVerifyNative]::ExpectedSha256($Pattern, $Bytes) } + +# 交给客体小程序(dataverify-guest.go)的计划文件:每块一行 `<偏移> <大小> <0|1>`(1 = O_DIRECT 写) +function New-GuestPlanText($Plan) { + (@($Plan | ForEach-Object { "$($_.Offset) $($_.Size) $(if ($_.Direct) { 1 } else { 0 })" }) -join "`n") + "`n" +} +# 核对:写 -> 重启前回读 -> sync、正常关机、冷启动 -> 回读 -> 删。 +# $Ops(脚本块,调用方给): +# Prepare 建客体里的目录、把花样(Pattern)放进去 +# Write $PlanText 按计划文件的内容写数据文件,成功返回 $true +# Hash 客体里数据文件的 SHA-256(小写十六进制),取不到返回 $null;这一步前面已经 sync 并清了页缓存 +# FreeBytes /data 剩余字节 +# Pull $LocalPath 把客体里的数据文件拉到宿主(出错时定位第一个不同的偏移) +# Reboot sync、正常关机、冷启动,返回 $true +# BootId 客体的 boot_id(核对真的重启过) +# Remove 删掉客体里的数据文件、花样、计划、小程序 +# 返回 [pscustomobject]@{ Problems; Notes } +function Invoke-DataVerify([int]$SizeMiB, [uint64]$Seed, [int]$DirectPercent, [hashtable]$Ops, [scriptblock]$Say = { param($m) }) { + $problems = @(); $notes = @() + $bytes = [int64]$SizeMiB * 1MB + $pattern = New-DataPattern $Seed + $plan = @(New-WritePlan $Seed $SizeMiB $DirectPercent) + $direct = @($plan | Where-Object Direct) + $sizeSet = @($plan.Size | Sort-Object -Unique) + $expected = Get-ExpectedSha256 $pattern $bytes + & $Say ("计划:{0} MiB,{1} 次写入(直写 {2} 次,普通写 {3} 次),单次 {4}–{5} KiB,期望 SHA-256 {6}" -f $SizeMiB, $plan.Count, $direct.Count, ($plan.Count - $direct.Count), ($sizeSet[0] / 1KB), ($sizeSet[-1] / 1KB), $expected.Substring(0, 12)) + $notes += "$SizeMiB MiB、$($plan.Count) 次写入(直写 $($direct.Count))" + try { + $free = & $Ops.FreeBytes + if ($null -ne $free -and $free -lt $bytes + 256MB) { throw "客体 /data 只剩 $([math]::Round($free / 1MB)) MiB,写 $SizeMiB MiB 不够(要多留 256 MiB)" } + & $Ops.Prepare $pattern + $t0 = Get-Date + if (-not (& $Ops.Write (New-GuestPlanText $plan))) { throw '客体里写花样文件失败' } + $notes += ('写 {0:N0} s' -f ((Get-Date) - $t0).TotalSeconds) + $bootBefore = & $Ops.BootId + $h = & $Ops.Hash + if ($h -ne $expected) { $problems += "重启前回读的哈希和期望的不一致(客体 $h,期望 $expected)" } + else { & $Say '重启前回读:哈希一致' } + $t0 = Get-Date + if (-not (& $Ops.Reboot)) { throw '关机再冷启动失败' } + $notes += ('关机加冷启动 {0:N0} s' -f ((Get-Date) - $t0).TotalSeconds) + $bootAfter = & $Ops.BootId + if ($bootBefore -and $bootBefore -eq $bootAfter) { $problems += "boot_id 没变($bootAfter):没有真的重启" } + $h = & $Ops.Hash + if ($h -eq $expected) { + & $Say '冷启动后回读:哈希一致,0 个错字节' + $notes += '冷启动后 0 个错字节' + } else { + $tmp = Join-Path ([IO.Path]::GetTempPath()) "dataverify-$PID.bin" + try { + & $Ops.Pull $tmp + $d = [DataVerifyNative]::Diff($pattern, $tmp, $bytes) + $where = if ($d[1] -gt 0) { "第一个不同的偏移 $($d[0])($([math]::Floor($d[0] / 4096)) 号 4 KiB 块),共 $($d[1]) 个错字节,文件大小 $($d[2])(应为 $bytes)" } else { "拉回来逐字节比对没有差异,但客体算的哈希是 $h" } + $problems += "冷启动后回读不一致:$where" + } catch { + $problems += "冷启动后回读的哈希不一致(客体 $h,期望 $expected),拉回来定位偏移也失败:$($_.Exception.Message)" + } finally { if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force } } + } + } catch { + $problems += "出错:$($_.Exception.Message)" + } finally { + try { & $Ops.Remove } catch { $problems += "清理客体里的文件失败:$($_.Exception.Message)" } + } + [pscustomobject]@{ Problems = $problems; Notes = $notes } +} diff --git a/items/dataverify.ps1 b/items/dataverify.ps1 new file mode 100644 index 0000000..f3cf4eb --- /dev/null +++ b/items/dataverify.ps1 @@ -0,0 +1,103 @@ +# 回归项:磁盘读写数据核对(ModAVD/ModAVD#138,#126 的常驻回归)。用测试根的 modavd 新建一台临时实例,在客体 /data 里写一个 +# 固定花样(由种子重新算得出来)的文件,总量默认 1 GiB(-SizeMiB 改小):单次写入 4 KiB–1 MiB,普通写和 O_DIRECT 写各占一半左右。 +# 写完 sync、清页缓存,回读一遍哈希;再 sync、正常关机、冷启动,回读哈希和宿主按种子算的比对;不一致就把文件拉回来, +# 报第一个不同的偏移和错字节数。最后删掉文件,再删掉这台实例(-Keep 留着实例)。 +# +# 写在客体的 /data/unencrypted/ 下:/data/local/tmp、/sdcard 是文件级加密的目录,ext4 在那里会把 O_DIRECT 悄悄退回普通写, +# 测不到直写路径。客体自带的 toybox dd 没有 oflag=direct,所以写入用 items\dataverify-guest.go 这个小程序(宿主上 go build 成静态 linux/amd64,推进去跑; +# 要宿主有 go),花样文件和写入计划也从宿主推进去;哈希用客体的 sha256sum。 +# 主体在 dataverify-lib.ps1(Invoke-DataVerify),tests\dataverify.ps1 换上假的客体测它。 +param( + [Parameter(Mandatory)][string]$Sdk, + # 测试用的 ModAVD 根目录(本机配置 TestRoot) + [string]$Root, + # 临时实例的编号;测试根里已经有这个编号就停,换一个。5 号 modavd 不让用 + [int]$Index = 4, + # 总量(MiB),默认 1024 = 1 GiB + [ValidateRange(1, 65536)][int]$SizeMiB = 1024, + # 花样的种子;同一个种子花样和写入计划都一样 + [uint64]$Seed = 0x4d4156444154, + # 直写(O_DIRECT)占写入次数的百分比 + [ValidateRange(0, 100)][int]$DirectPercent = 50, + [switch]$Keep +) +$ErrorActionPreference = 'Stop' +[cultureinfo]::CurrentCulture = [cultureinfo]::InvariantCulture +. "$PSScriptRoot\common.ps1" +. "$PSScriptRoot\dataverify-lib.ps1" +$Root = Resolve-TestRoot (Use-Local TestRoot $Root '-Root') + +$guestDir = '/data/unencrypted/e2e-dv' +$work = Join-Path ([IO.Path]::GetTempPath()) "dataverify-$PID" + +$ops = @{ + FreeBytes = { + $o = Su 'df -k /data | tail -n 1' + $f = @("$($o | Select-Object -Last 1)".Trim() -split '\s+') + if ($f.Count -ge 4) { [int64]$f[3] * 1KB } else { $null } + } + Prepare = { + param($pattern) + New-Item -ItemType Directory -Force $work | Out-Null + $go = Get-Command go -ErrorAction SilentlyContinue + if (-not $go) { throw '宿主上没有 go:客体里的写入小程序(items\dataverify-guest.go)要用它编(客体的 dd 没有 oflag=direct)' } + $env:GOOS = 'linux'; $env:GOARCH = 'amd64'; $env:CGO_ENABLED = '0' + try { $b = & $go.Source build -trimpath -o (Join-Path $work 'dvwrite') (Join-Path $PSScriptRoot 'dataverify-guest.go') 2>&1 } + finally { Remove-Item Env:GOOS, Env:GOARCH, Env:CGO_ENABLED -ErrorAction SilentlyContinue } + if ($LASTEXITCODE -ne 0) { throw "编客体小程序失败:$($b -join ' ')" } + [IO.File]::WriteAllBytes((Join-Path $work 'pattern.bin'), $pattern) + A push (Join-Path $work 'pattern.bin') /data/local/tmp/dv-pattern.bin | Out-Null + A push (Join-Path $work 'dvwrite') /data/local/tmp/dvwrite | Out-Null + $o = Su "chmod 755 /data/local/tmp/dvwrite && mkdir -p $guestDir && cp /data/local/tmp/dv-pattern.bin $guestDir/pattern.bin && rm -f /data/local/tmp/dv-pattern.bin && echo DV-PREP-OK" + if (-not ($o -match 'DV-PREP-OK')) { throw "客体里准备目录和花样失败:$($o -join ' ')" } + } + Write = { + param($text) + $f = Join-Path $work 'plan.txt' + [IO.File]::WriteAllText($f, ($text -replace "`r`n", "`n")) + A push $f /data/local/tmp/dv-plan.txt | Out-Null + $o = Su "cp /data/local/tmp/dv-plan.txt $guestDir/plan.txt && rm -f /data/local/tmp/dv-plan.txt && /data/local/tmp/dvwrite $guestDir/pattern.bin $guestDir/dv.bin $guestDir/plan.txt" + $o | Where-Object { $_ -match 'DV-WRITE' } | ForEach-Object { Write-Host "$(Get-Date -Format 'HH:mm:ss') $_" } + [bool]($o -match 'DV-WRITE-OK') + } + Hash = { + $o = Su "sync; echo 3 > /proc/sys/vm/drop_caches; sha256sum $guestDir/dv.bin" + $m = [regex]::Match(($o -join "`n"), '(?m)^([0-9a-f]{64})\s') + if ($m.Success) { $m.Groups[1].Value } else { $null } + } + Pull = { param($local) A pull "$guestDir/dv.bin" $local | Out-Null } + BootId = { "$(Su 'cat /proc/sys/kernel/random/boot_id' | Select-Object -Last 1)".Trim() } + Reboot = { + Su 'sync' | Out-Null + M stop $Index | Select-Object -Last 1 | ForEach-Object { Write-Host "$(Get-Date -Format 'HH:mm:ss') $_" } + if ($LASTEXITCODE -ne 0) { return $false } + M start $Index | Select-Object -Last 1 | ForEach-Object { Write-Host "$(Get-Date -Format 'HH:mm:ss') $_" } + $LASTEXITCODE -eq 0 + } + Remove = { Su "rm -rf $guestDir /data/local/tmp/dvwrite /data/local/tmp/dv-plan.txt /data/local/tmp/dv-pattern.bin" | Out-Null } +} + +$problems = @(); $notes = @(); $created = $false +$old = Set-RootSdk $Root $Sdk +try { + M info $Index | Out-Null + if ($LASTEXITCODE -eq 0) { throw "测试根里已经有实例 $Index,换 -Index 或先删掉它" } + $o = M create --index $Index --title 回归数据核对 --memory 4G --cpu 4 --data-gb 16 --no-bridge + if ($LASTEXITCODE -ne 0) { throw "新建失败:$($o -join ' ')" } + $created = $true + M start $Index | Select-Object -Last 2 | ForEach-Object { Write-Host "$(Get-Date -Format 'HH:mm:ss') $_" } + if ($LASTEXITCODE -ne 0) { throw "首次开机失败(modavd 退出码 $LASTEXITCODE)" } + $r = Invoke-DataVerify -SizeMiB $SizeMiB -Seed $Seed -DirectPercent $DirectPercent -Ops $ops -Say { param($m) Write-Host "$(Get-Date -Format 'HH:mm:ss') $m" } + $problems += $r.Problems; $notes += $r.Notes +} catch { + $problems += "出错:$($_.Exception.Message)" +} finally { + M stop $Index | Select-Object -Last 1 | ForEach-Object { Write-Host "$(Get-Date -Format 'HH:mm:ss') $_" } + if ($created -and -not $Keep) { Remove-TestInstance $Index } + Set-RootSdk $Root $old | Out-Null + if (Test-Path -LiteralPath $work) { Remove-Item -LiteralPath $work -Recurse -Force } +} +$summary = $notes -join ';' +if ($problems) { $summary += ";问题:$($problems -join ';')" } +Result ($problems.Count -eq 0) $summary +exit 0 diff --git a/suites/release.psd1 b/suites/release.psd1 index 1c84e94..c6ee4d7 100644 --- a/suites/release.psd1 +++ b/suites/release.psd1 @@ -105,6 +105,13 @@ Args = @('-Sdk', '{Sdk}') TimeoutMin = 20 } + @{ + Id = 'dataverify'; Title = '磁盘读写数据核对(ModAVD/ModAVD#138)'; Criterion = '客体写 1 GiB 固定花样(单次 4 KiB–1 MiB,普通写和 O_DIRECT 混着),sync、正常关机、冷启动后回读,0 个错字节' + Scopes = @('emulator', 'manager') + Script = '{Repo}\items\dataverify.ps1' + Args = @('-Sdk', '{Sdk}') + TimeoutMin = 25 + } @{ Id = 'upgrade'; Title = '从上一个生产版本升级(照「怎么更新」整个覆盖新包)'; Criterion = '旧版建的实例照常开机、数据都在;版本号和包根新文件到位;旧盘扩满(0.8.0 起自动扩到 256 GB,以前改大到 80 GB);升级后 modavd 照常开机自动装好执行桥' # 只改输入法、只重新打包时打包前不跑这项:程序没变,打完包用组包目录跑的那次(PostPackage)才有意义 diff --git a/tests/dataverify.ps1 b/tests/dataverify.ps1 new file mode 100644 index 0000000..550327c --- /dev/null +++ b/tests/dataverify.ps1 @@ -0,0 +1,125 @@ +# dataverify 项的测试:不开模拟器、不碰 adb 和 modavd。用宿主上的一个临时文件冒充客体里的数据文件, +# 假的 Write 把计划文件逐行"执行"到这个文件上(和客体小程序同样的算法:偏移 o 处写花样[o mod 长度]起的一段),所以同时核对了计划 -> 计划文件 -> 文件内容这条链。 +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot\..\items\dataverify-lib.ps1" +$count = 0 +function Check([bool]$ok, [string]$message) { + $script:count++ + if (-not $ok) { throw $message } +} + +# 花样和计划可复现,换种子就不同 +$a = New-DataPattern 42; $b = New-DataPattern 42; $c = New-DataPattern 43 +Check ($a.Length -eq 7MB) 'pattern length' +Check ([Linq.Enumerable]::SequenceEqual([byte[]]$a, [byte[]]$b)) 'same seed, same pattern' +Check (-not [Linq.Enumerable]::SequenceEqual([byte[]]$a, [byte[]]$c)) 'different seed, different pattern' +$p1 = @(New-WritePlan 42 64); $p2 = @(New-WritePlan 42 64); $p3 = @(New-WritePlan 43 64) +Check (($p1 | ConvertTo-Json -Compress) -eq ($p2 | ConvertTo-Json -Compress)) 'same seed, same plan' +Check (($p1 | ConvertTo-Json -Compress) -ne ($p3 | ConvertTo-Json -Compress)) 'different seed, different plan' + +# 计划:覆盖整个文件、不重叠、按偏移递增,大小在 4 KiB–1 MiB 之间且是自己偏移的约数,普通写和直写都有 +$plan = @(New-WritePlan 7 256) +$pos = 0L; $ok = $true +foreach ($x in $plan) { if ($x.Offset -ne $pos -or $x.Size -lt 4096 -or $x.Size -gt 1MB -or $x.Offset % $x.Size -ne 0) { $ok = $false }; $pos += $x.Size } +Check ($ok -and $pos -eq 256MB) 'plan covers the file exactly with aligned 4 KiB-1 MiB chunks' +Check (@($plan | Where-Object Direct).Count -gt 0 -and @($plan | Where-Object { -not $_.Direct }).Count -gt 0) 'plan mixes buffered and direct' +Check (@($plan.Size | Sort-Object -Unique).Count -ge 6) 'plan mixes many sizes' +Check (@(New-WritePlan 7 8 0 | Where-Object Direct).Count -eq 0 -and @(New-WritePlan 7 8 100 | Where-Object { -not $_.Direct }).Count -eq 0) 'direct percent 0 / 100' +$failed = $false; try { New-WritePlan 1 0 | Out-Null } catch { $failed = $true } +Check $failed 'size 0 must fail' + +# 计划文件(交给客体小程序):每块一行 `偏移 大小 0|1`,直写的行末尾是 1 +$text = New-GuestPlanText $plan +$lines = @($text -split "`n" | Where-Object { $_ }) +Check ($lines.Count -eq $plan.Count -and $lines[0] -match '^0 \d+ [01]$') 'one plan line per chunk' +Check (@($lines | Where-Object { $_ -match ' 1$' }).Count -eq @($plan | Where-Object Direct).Count) 'direct lines end with 1' +# 假客体:一个宿主上的文件 +$dir = Join-Path ([IO.Path]::GetTempPath()) ("dataverify-test-" + [guid]::NewGuid()) +New-Item -ItemType Directory $dir | Out-Null +try { + function New-FakeOps([string]$Dir, [hashtable]$State) { + $file = Join-Path $Dir 'guest-dv.bin' + $ops = @{} + $ops.FreeBytes = { if ($null -ne $State.Free) { $State.Free } else { 100GB } }.GetNewClosure() + $ops.Prepare = { param($pattern) $State.Pattern = $pattern; $State.Prepared = $true }.GetNewClosure() + $ops.Write = { + param($text) + $pattern = $State.Pattern + $fs = [IO.File]::Open($file, 'Create', 'ReadWrite') + try { + foreach ($l in ($text -split "`n")) { + if ($l -notmatch '^(\d+) (\d+) ([01])$') { continue } + $off = [int64]$Matches[1]; $size = [int]$Matches[2] + $fs.Position = $off + $fs.Write($pattern, [int]($off % $pattern.Length), $size) + if ($Matches[3] -eq '1') { $State.Direct++ } else { $State.Buffered++ } + } + } finally { $fs.Dispose() } + $true + }.GetNewClosure() + $ops.BootId = { "boot-$($State.Boots)" }.GetNewClosure() + $ops.Hash = { + if (-not (Test-Path -LiteralPath $file)) { return $null } + (Get-FileHash -LiteralPath $file -Algorithm SHA256).Hash.ToLowerInvariant() + }.GetNewClosure() + $ops.Pull = { param($local) Copy-Item -LiteralPath $file -Destination $local -Force }.GetNewClosure() + $ops.Reboot = { + $State.Boots++ + if ($State.Corrupt -is [scriptblock]) { & $State.Corrupt $file } + -not $State.RebootFails + }.GetNewClosure() + $ops.Remove = { $State.Removed = $true; if (Test-Path -LiteralPath $file) { Remove-Item -LiteralPath $file -Force } }.GetNewClosure() + $ops + } + function New-State { @{ Boots = 1; Direct = 0; Buffered = 0; Removed = $false } } + + # 1. 正常:通过,写了直写和普通写,删干净 + $s = New-State + $r = Invoke-DataVerify 8 99 50 (New-FakeOps $dir $s) + Check ($r.Problems.Count -eq 0) "clean run must pass: $($r.Problems -join '; ')" + Check ($s.Direct -gt 0 -and $s.Buffered -gt 0 -and $s.Removed -and -not (Test-Path (Join-Path $dir 'guest-dv.bin'))) 'both write kinds issued and cleaned up' + Check (($r.Notes -join ' ') -match '0 个错字节') 'notes report zero bad bytes' + + # 2. 冷启动后翻一个字节:失败,报偏移 1234567 和 1 个错字节,仍然清理 + $s = New-State + $s.Corrupt = { param($f) $fs = [IO.File]::Open($f, 'Open', 'ReadWrite'); try { $fs.Position = 1234567; $b = $fs.ReadByte(); $fs.Position = 1234567; $fs.WriteByte(($b -bxor 0xff)) } finally { $fs.Dispose() } } + $r = Invoke-DataVerify 8 99 50 (New-FakeOps $dir $s) + Check ($r.Problems.Count -eq 1 -and $r.Problems[0] -match '第一个不同的偏移 1234567' -and $r.Problems[0] -match '共 1 个错字节') "one flipped byte must be located: $($r.Problems -join '; ')" + Check $s.Removed 'cleanup after mismatch' + + # 3. 冷启动后少了尾巴(截短):失败,错字节数 = 少的字节数 + $s = New-State + $s.Corrupt = { param($f) $fs = [IO.File]::Open($f, 'Open', 'ReadWrite'); try { $fs.SetLength(8MB - 4096) } finally { $fs.Dispose() } } + $r = Invoke-DataVerify 8 99 50 (New-FakeOps $dir $s) + Check ($r.Problems.Count -eq 1 -and $r.Problems[0] -match '第一个不同的偏移 8384512' -and $r.Problems[0] -match '共 4096 个错字节') "truncation must be reported: $($r.Problems -join '; ')" + + # 4. 冷启动后文件没了(哈希取不到):失败 + $s = New-State + $s.Corrupt = { param($f) Remove-Item -LiteralPath $f -Force } + $r = Invoke-DataVerify 8 99 50 (New-FakeOps $dir $s) + Check ($r.Problems.Count -ge 1) 'missing file must fail' + Check ($s.Removed) 'cleanup after missing file' + + # 5. 关机或冷启动失败:失败并清理;写入失败、剩余空间不够:失败并清理,且不写 + $s = New-State; $s.RebootFails = $true + $r = Invoke-DataVerify 8 99 50 (New-FakeOps $dir $s) + Check ($r.Problems.Count -ge 1 -and $r.Problems[0] -match '关机再冷启动失败' -and $s.Removed) 'reboot failure must fail and clean up' + $s = New-State; $s.Free = 4MB + $r = Invoke-DataVerify 8 99 50 (New-FakeOps $dir $s) + Check ($r.Problems.Count -eq 1 -and $r.Problems[0] -match '不够' -and $s.Direct -eq 0 -and $s.Buffered -eq 0 -and $s.Removed) 'not enough free space: fail before writing' + + # 6. boot_id 没变:没有真的重启,失败 + $s = New-State + $ops = New-FakeOps $dir $s + $ops.BootId = { 'same' } + $r = Invoke-DataVerify 8 99 50 $ops + Check ($r.Problems.Count -eq 1 -and $r.Problems[0] -match 'boot_id 没变') 'unchanged boot_id must fail' + + # 7. 清理本身出错也要报 + $s = New-State + $ops = New-FakeOps $dir $s + $ops.Remove = { throw 'rm failed' } + $r = Invoke-DataVerify 8 99 50 $ops + Check ($r.Problems.Count -eq 1 -and $r.Problems[0] -match '清理客体里的文件失败') 'cleanup failure must be reported' +} finally { Remove-Item -LiteralPath $dir -Recurse -Force } +Write-Host "$count assertions passed"