diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index e452a3792..12ef6fd4b 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -768,9 +768,21 @@ jobs: with: programs: '["debug"]' + # x86_64 Android, against the Windows package (which ships Android's libraries): the tester's + # tests and the default library's, under every memory model, built with the package's compiler + # and run on an x86_64 emulator. The other ABIs are linked only (test-compile-android-link). + test_android: + name: Android emulator + needs: [defaultlib_commit, build_on_windows] + uses: ./.github/workflows/test-android.yml + with: + package: artifact + defaultlib_sha: ${{ needs.defaultlib_commit.outputs.sha }} + create_release: needs: [build_on_windows, build_on_linux, test_defaultlib_windows, test_defaultlib_linux, - test_wasm_windows, test_wasm_linux, test_debug_programs_windows, test_debug_programs_linux] + test_wasm_windows, test_wasm_linux, test_debug_programs_windows, test_debug_programs_linux, + test_android] name: Create Release runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test-android.yml b/.github/workflows/test-android.yml new file mode 100644 index 000000000..42e739d2a --- /dev/null +++ b/.github/workflows/test-android.yml @@ -0,0 +1,157 @@ +name: Test - Android (emulator) + +# The tester's tests and the default library's, built for x86_64 Android (API 29) under -mm=gc, rc, +# none and own with a Windows package's compiler and Android libraries, and run on an x86_64 Android +# emulator (tslang/test/android/run_tests.mjs). The package ships Android's libraries only for +# Windows, so Windows builds the programs and Linux, which has KVM, runs the emulator. +# +# create-release.yml calls it with the package it has just built (package: artifact). A pull request +# that changes the runner, and a manual run, test the latest published release's package. + +on: + workflow_call: + inputs: + package: + description: "artifact: this run's tslang-windows-asset; release: the latest published release's Windows package" + type: string + default: release + defaultlib_sha: + description: the TypeScriptCompilerDefaultLib commit the package's default library was built from (default its main) + type: string + default: '' + pull_request: + branches: [ main ] + paths: + - 'tslang/test/android/**' + - 'tslang/test/suites.mjs' + - '.github/workflows/test-android.yml' + workflow_dispatch: + +jobs: + build: + name: Build Android tests (x86_64) + runs-on: windows-2025-vs2026 + timeout-minutes: 60 + env: + TOOL_PATH: ${{ github.workspace }}\__tslang + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Clone Default Library + shell: pwsh + env: + DEFAULTLIB_SHA: ${{ inputs.defaultlib_sha }} + run: | + git clone https://github.com/ASDAlexander77/TypeScriptCompilerDefaultLib.git + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ($env:DEFAULTLIB_SHA) { + git -C TypeScriptCompilerDefaultLib checkout --detach $env:DEFAULTLIB_SHA + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + + - name: Download the package (this run's) + if: inputs.package == 'artifact' + uses: actions/download-artifact@v4 + with: + name: tslang-windows-asset + + - name: Download the package (the latest release's) + if: inputs.package != 'artifact' + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download --repo ${{ github.repository }} --pattern '*-windows-x64.zip' + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Unpack the package + shell: pwsh + run: | + $zip = Get-ChildItem -Filter '*-windows-x64.zip' | Select-Object -First 1 + Write-Host "package: $($zip.Name)" + Expand-Archive -Path $zip.FullName -DestinationPath $env:TOOL_PATH + Remove-Item $zip.FullName + New-Item -ItemType Directory -Force -Path TypeScriptCompilerDefaultLib\__build | Out-Null + Move-Item -Path (Join-Path $env:TOOL_PATH 'defaultlib') -Destination TypeScriptCompilerDefaultLib\__build\defaultlib -ErrorAction Stop + & (Join-Path $env:TOOL_PATH 'tslang.exe') --version + + - name: Build the tests for x86_64 Android + shell: pwsh + working-directory: ${{ github.workspace }}/tslang/test/android + run: | + $ndk = if ($env:ANDROID_NDK_LATEST_HOME) { $env:ANDROID_NDK_LATEST_HOME } else { $env:ANDROID_NDK_HOME } + if (-not $ndk) { Write-Error "no Android NDK on this runner (ANDROID_NDK_LATEST_HOME / ANDROID_NDK_HOME)"; exit 1 } + node run_tests.mjs --build "--tslang=$(Join-Path $env:TOOL_PATH 'tslang.exe')" ` + "--default-lib-repo=${{ github.workspace }}\TypeScriptCompilerDefaultLib" ` + "--android-lib=$(Join-Path $env:TOOL_PATH 'android\x86_64\lib')" "--android-ndk-path=$ndk" -j4 + $code = $LASTEXITCODE + if ($code -ne 0) { Get-Content out\results.txt -ErrorAction SilentlyContinue } + tar -czf android-tests.tgz -C out . + exit $code + + - name: Upload the tests + uses: actions/upload-artifact@v4 + with: + name: android-tests + path: tslang/test/android/android-tests.tgz + retention-days: 1 + + run: + name: Run Android tests (x86_64 emulator, API 29) + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Download the tests + uses: actions/download-artifact@v4 + with: + name: android-tests + path: tslang/test/android + + - name: Unpack the tests + working-directory: ${{ github.workspace }}/tslang/test/android + shell: bash + run: mkdir -p out && tar -xzf android-tests.tgz -C out && rm android-tests.tgz && du -sh out + + # the emulator needs KVM, which the runner has but does not open to its user + - name: Enable KVM + shell: bash + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Run the tests on the emulator + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 29 + arch: x86_64 + target: default + cores: 4 + ram-size: 4096M + disable-animations: true + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -no-snapshot + script: cd tslang/test/android && node run_tests.mjs --run --adb=adb || { cat out/results.txt; exit 1; } + + - name: Keep the results + if: always() + uses: actions/upload-artifact@v4 + with: + name: android-test-results + path: tslang/test/android/out/results.txt + retention-days: 7 + if-no-files-found: ignore diff --git a/tslang/test/android/.gitignore b/tslang/test/android/.gitignore new file mode 100644 index 000000000..910c1e725 --- /dev/null +++ b/tslang/test/android/.gitignore @@ -0,0 +1 @@ +out*/ diff --git a/tslang/test/android/README.md b/tslang/test/android/README.md new file mode 100644 index 000000000..33b0f175a --- /dev/null +++ b/tslang/test/android/README.md @@ -0,0 +1,43 @@ +# Running the tests on Android + +[run_tests.mjs](run_tests.mjs) builds the tester's tests and the default library's tests for +Android, and runs them on a device or an emulator through adb. Which tests there are, and when one +passes, is shared with the wasm tests: [../suites.mjs](../suites.mjs). + + node run_tests.mjs [--build | --run] [--suite=tester,defaultlib] [--mm=gc,rc,none,own] [--abi=x86_64] [-j] [filter] + +- `--build` compiles every test with `-mtriple= --opt` and links it through + the NDK (`ANDROID_NDK_HOME` or `--android-ndk-path`). It uses the Android default library, the + collector and the async runtime: `scripts\build_gc_release_android.bat`, + `scripts\build_tslang_runtime_release_android.bat` and the default library's + `scripts\build_android.bat`, or a release package's (`--android-lib=\android\\lib`). + Each program is stripped. The programs and `manifest.json` go to `out/`. +- `--run` needs only adb and `out/`. It runs one leg (a suite under a model) at a time in + `/data/local/tmp/tslang-tests`: it pushes the leg's programs, runs them on the device four at a + time, each under `timeout` and a memory limit, and pulls back each one's exit code and the last + lines of its output. The summary and every failure are written to `out/results.txt`. +- With neither flag it does both. + +The tests in [android_skips.txt](android_skips.txt) are skipped, each with its reason. So are the +default library's tests marked `// gc only` (except under gc) or `// not own` (under own). Under +`-mm=own`, a test that does not compile for the host either is listed as NATIVE, as in the wasm +runner. + +## An emulator + +An x86_64 system image runs the x86_64 programs natively. For example, with the Android SDK: + + sdkmanager "system-images;android-29;default;x86_64" "emulator" "platform-tools" + avdmanager create avd -n tslang_api29_x86_64 -k "system-images;android-29;default;x86_64" + emulator -avd tslang_api29_x86_64 -no-window -no-audio -no-boot-anim -no-snapshot -memory 2048 + +arm64-v8a and armeabi-v7a programs need an arm device: an x86_64 emulator runs them only through +its binary translator, which is not reliable enough to test with. + +## In CI + +[test-android.yml](../../../.github/workflows/test-android.yml) builds the x86_64 programs on +Windows, with a Windows package's compiler and Android libraries (the only package that ships +them), and runs them on an API 29 x86_64 emulator on Linux. A release runs it against the package +it has just built (`test_android` in `create-release.yml`); a pull request that changes the runner +runs it against the latest published release. diff --git a/tslang/test/android/android_skips.txt b/tslang/test/android/android_skips.txt new file mode 100644 index 000000000..da56f0325 --- /dev/null +++ b/tslang/test/android/android_skips.txt @@ -0,0 +1,10 @@ +# Tests run_tests.mjs does not run on Android: ` [,...] # `, every leg +# when none is given. + +# cannot work on Android +00cppvtable_hidden_return.ts # its hand-made vtable slots return a record the MSVC way: Windows only, as in ctest +fetch.ts # (default library) fetch(): Android's default library has no HTTP (no libcurl in the NDK), so the request throws +json.ts none # (default library) builds a 100000-character string a character at a time: -mm=none frees none of the ~5 GB + +# fails natively under the same model too (not run natively under it by ctest) +00global_interface_initializer.ts own # asserts "a field read" (native own: the same assert) diff --git a/tslang/test/android/run_tests.mjs b/tslang/test/android/run_tests.mjs new file mode 100644 index 000000000..2856bb093 --- /dev/null +++ b/tslang/test/android/run_tests.mjs @@ -0,0 +1,286 @@ +// Compiles tests for Android and runs them on a device or emulator through adb. +// +// node run_tests.mjs [options] [filter] +// +// --build only compile: the programs and a manifest go to --out +// --run only run what --build left in --out (needs adb, not the compiler) +// (neither: both) +// --suite=tester,defaultlib which tests (default: both) - see ../suites.mjs +// --mm=gc,rc,none,own memory models (default: all four) +// --abi=x86_64 arm64-v8a, armeabi-v7a or x86_64 (default: x86_64, which an x86_64 +// emulator runs natively) +// --tslang= tslang (default: the release build tree, else tslang on PATH) +// --default-lib-repo= the default library's repository, for its tests (default: +// ../TypeScriptCompilerDefaultLib next to this one) +// --default-lib-path= the folder holding defaultlib/ (default: /__build) +// --android-lib= the folder holding the Android libgc.a and libTypeScriptAsyncRuntime.a, +// as a release package has them: android//lib (default: this +// repository's 3rdParty/gc/android//release/lib and +// __build/tslang-runtime/release/android/) +// --android-ndk-path= the NDK tslang links with (default: ANDROID_NDK_HOME) +// -j compiles at a time (default: the number of CPUs) +// --device-jobs= programs at a time on the device (default: 4) +// --timeout= per program run (default: 120) +// --out= programs, manifest.json and results.txt (default: ./out) +// --adb= adb (default: adb on PATH) +// filter a regular expression the test's file name has to match +// +// A test passes as ../suites.mjs says. Skipped, and listed as such: the tests in android_skips.txt +// and the default library's tests marked "// gc only" (but under gc) or (under own) "// not own". +// Under -mm=own, a test that does not compile for Android and does not compile for the host either is +// listed apart (NATIVE), not counted as a failure - what -mm=own cannot prove yet. +// +// On the device each program runs in /data/local/tmp/tslang-tests under `timeout` and a memory +// limit (an out-of-memory adb-shell process can take the emulator down), with its output in files; +// what comes back is its exit code, whether it printed "done." / "ALL DONE", how much it wrote to +// stderr and the last lines of each. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { repo, makeSuites, loadSkips, skipReason, runProcess, lastLines, firstLines, report, runJobs } from "../suites.mjs"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const ABIS = { + "arm64-v8a": "aarch64-linux-android29", + "armeabi-v7a": "armv7a-linux-androideabi29", + "x86_64": "x86_64-linux-android29", +}; + +const options = { + suite: ["tester", "defaultlib"], mm: ["gc", "rc", "none", "own"], abi: "x86_64", build: false, run: false, + jobs: os.cpus().length, deviceJobs: 4, timeout: 120, out: path.join(here, "out"), adb: "adb", filter: null, +}; +for (const arg of process.argv.slice(2)) +{ + let m; + if (arg === "--build") options.build = true; + else if (arg === "--run") options.run = true; + else if ((m = /^--suite=(.+)$/.exec(arg))) options.suite = m[1].split(","); + else if ((m = /^--mm=(.+)$/.exec(arg))) options.mm = m[1].split(","); + else if ((m = /^--abi=(.+)$/.exec(arg))) options.abi = m[1]; + else if ((m = /^--tslang=(.+)$/.exec(arg))) options.tslang = m[1]; + else if ((m = /^--default-lib-repo=(.+)$/.exec(arg))) options.defaultLibRepo = path.resolve(m[1]); + else if ((m = /^--default-lib-path=(.+)$/.exec(arg))) options.defaultLibPath = path.resolve(m[1]); + else if ((m = /^--android-lib=(.+)$/.exec(arg))) options.androidLib = path.resolve(m[1]); + else if ((m = /^--android-ndk-path=(.+)$/.exec(arg))) options.ndk = path.resolve(m[1]); + else if ((m = /^-j(\d+)$/.exec(arg))) options.jobs = Number(m[1]); + else if ((m = /^--device-jobs=(\d+)$/.exec(arg))) options.deviceJobs = Number(m[1]); + else if ((m = /^--timeout=(\d+)$/.exec(arg))) options.timeout = Number(m[1]); + else if ((m = /^--out=(.+)$/.exec(arg))) options.out = path.resolve(m[1]); + else if ((m = /^--adb=(.+)$/.exec(arg))) options.adb = m[1]; + else if (!arg.startsWith("-")) options.filter = new RegExp(arg); + else { console.error(`unknown option ${arg}`); process.exit(2); } +} + +if (!options.build && !options.run) options.build = options.run = true; +const triple = ABIS[options.abi]; +if (!triple) { console.error(`unknown ABI ${options.abi}: ${Object.keys(ABIS).join(", ")}`); process.exit(2); } + +const manifestFile = path.join(options.out, "manifest.json"); + +// ---- build: compile every test, write the manifest ---- +const build = async () => +{ + const exe = process.platform === "win32" ? "tslang.exe" : "tslang"; + options.tslang ??= [ + path.join(repo, "__build/tslang/windows-msbuild-2026-release/bin", exe), + path.join(repo, "__build/tslang/ninja/release/bin", exe), + ].find((p) => fs.existsSync(p)) ?? exe; + options.defaultLibRepo ??= path.resolve(repo, "../TypeScriptCompilerDefaultLib"); + options.defaultLibPath ??= path.join(options.defaultLibRepo, "__build"); + const gcLib = options.androidLib ?? path.join(repo, "3rdParty/gc/android", options.abi, "release/lib"); + const runtimeLib = options.androidLib ?? path.join(repo, "__build/tslang-runtime/release/android", options.abi); + const ndk = options.ndk ?? process.env.ANDROID_NDK_HOME; + if (!ndk) { console.error("no Android NDK: pass --android-ndk-path or set ANDROID_NDK_HOME"); process.exit(2); } + // the programs carry the debug information of the collector, libc++ and the default library's + // C++ wrappers: four times their size, more than an artifact and an emulator should take + const host = { win32: "windows-x86_64", linux: "linux-x86_64", darwin: "darwin-x86_64" }[process.platform]; + const strip = path.join(ndk, "toolchains/llvm/prebuilt", host, "bin", process.platform === "win32" ? "llvm-strip.exe" : "llvm-strip"); + if (!fs.existsSync(strip)) { console.error(`no llvm-strip in the NDK: ${strip}`); process.exit(2); } + + const skips = loadSkips(path.join(here, "android_skips.txt"), (x) => x in ABIS); + const suites = makeSuites(options.defaultLibRepo); + // the environment the compiler looks the libraries up in, and nothing of the user's own lib paths + const compileEnv = { DEFAULT_LIB_PATH: "", GC_LIB_PATH: "", LLVM_LIB_PATH: "", TSLANG_LIB_PATH: "", ANDROID_NDK_HOME: ndk }; + const libArgs = [`--default-lib-path=${options.defaultLibPath}`, `--gc-lib-path=${gcLib}`, `--tslang-lib-path=${runtimeLib}`]; + + const legs = []; + for (const suiteName of options.suite) + { + if (!suites[suiteName]) { console.error(`unknown suite ${suiteName}: tester or defaultlib`); process.exit(2); } + for (const mm of options.mm) + legs.push({ suiteName, suite: suites[suiteName], target: options.abi, mm, linux: true, name: `${suiteName}-${options.abi}-${mm}` }); + } + + fs.rmSync(options.out, { recursive: true, force: true }); + const jobs = []; + const results = []; + for (const leg of legs) + { + fs.mkdirSync(path.join(options.out, leg.name), { recursive: true }); + for (const file of leg.suite.files().filter((f) => !options.filter || options.filter.test(f)).sort()) + { + const source = fs.readFileSync(path.join(leg.suite.dir, file), "utf8"); + const reason = skipReason(skips, leg.suite, file, source, leg); + if (reason) { results.push({ leg: leg.name, file, status: "SKIP", detail: reason }); continue; } + jobs.push({ leg, file }); + } + } + + const built = []; + const compile = async ({ leg, file }) => + { + const source = path.join(leg.suite.dir, file); + const name = file.replace(/\.ts$/, ""); + const binary = path.join(options.out, leg.name, name); + const compiled = await runProcess(options.tslang, ["--emit=exe", `-mm=${leg.mm}`, `-mtriple=${triple}`, "--opt", + ...leg.suite.compileArgs, ...libArgs, `-o=${binary}`, source], { env: compileEnv, timeout: 300 }); + if (compiled.code !== 0 || !fs.existsSync(binary)) + { + const detail = firstLines(compiled.stderr + compiled.stdout, 6); + if (leg.mm !== "own") return { leg: leg.name, file, status: "COMPILE", detail }; + + const obj = `${binary}.native.o`; + const native = await runProcess(options.tslang, ["--emit=obj", `-mm=${leg.mm}`, "--opt", ...leg.suite.compileArgs, + `-o=${obj}`, source], { env: compileEnv, timeout: 300 }); + fs.rmSync(obj, { force: true }); + if (native.code === 0) return { leg: leg.name, file, status: "COMPILE", detail }; + return { leg: leg.name, file, status: "NATIVE", detail: "native: " + firstLines(native.stderr + native.stdout, 4) }; + } + + const stripped = await runProcess(strip, [binary], { timeout: 120 }); + if (stripped.code !== 0) return { leg: leg.name, file, status: "COMPILE", detail: `llvm-strip: ${firstLines(stripped.stderr + stripped.stdout, 4)}` }; + + const stdin = source.replace(/\.ts$/, ".stdin"); + if (fs.existsSync(stdin)) fs.copyFileSync(stdin, `${binary}.stdin`); + built.push({ leg: leg.name, file, name, cwd: leg.suiteName === "defaultlib" ? "dl" : leg.name, stdin: fs.existsSync(stdin) }); + return { leg: leg.name, file, status: "BUILT" }; + }; + + console.log(`tslang: ${options.tslang}\ntarget: ${triple}\ndefault lib: ${options.defaultLibPath}\nlegs: ${legs.map((l) => l.name).join(", ")}; ${jobs.length} to build, ${results.length} skips`); + await runJobs(jobs, options.jobs, compile, results); + + // what is not to run: skipped, not compiled, NATIVE; the run step reports it with its own results + const manifest = { triple, legs: legs.map((l) => l.name), results: results.filter((r) => r.status !== "BUILT"), programs: built }; + fs.writeFileSync(manifestFile, JSON.stringify(manifest, null, 1)); + console.log(`built ${built.length} programs into ${options.out}`); + return manifest; +}; + +// ---- run: push the programs, run them on the device, judge what comes back ---- +const DEVICE = "/data/local/tmp/tslang-tests"; + +// one line a program: / . Each of the jobs takes every jobs-th line. +const deviceScript = (timeout, jobs) => `#!/system/bin/sh +D=${DEVICE} +cd $D || exit 1 +mkdir -p res +runone() { + id=$1; bin=$2; wd=$3; in=/dev/null + [ "$4" = 1 ] && in=$D/$bin.stdin + cd $D/$wd + # an adb-shell process the kernel kills for memory takes the whole emulator down: limit it + # toybox's timeout (API 29) exits with 124 for a program a signal ended too, not only for one it + # timed out: the seconds it ran tell them apart + start=$(date +%s) + ( ulimit -v 2097152; timeout ${timeout} $D/$bin < $in > $D/res/$id.out 2> $D/res/$id.err ); code=$? + secs=$(( $(date +%s) - start )) + done=0; grep -q 'done\\.' $D/res/$id.out $D/res/$id.err 2>/dev/null && done=1 + all=0; grep -q 'ALL DONE' $D/res/$id.out 2>/dev/null && all=1 + errbytes=$(wc -c < $D/res/$id.err) + tail -c 2000 $D/res/$id.out > $D/res/$id.otail; tail -c 2000 $D/res/$id.err > $D/res/$id.etail + rm -f $D/res/$id.out $D/res/$id.err + echo "$code $done $all $errbytes $secs" > $D/res/$id.sum +} +worker() { + n=0 + while read id bin wd in; do + [ $((n % ${jobs})) = $1 ] && runone $id $bin $wd $in + n=$((n + 1)) + done < $D/programs.txt +} +k=0 +while [ $k -lt ${jobs} ]; do worker $k & k=$((k + 1)); done +wait +echo finished +`; + +const adb = (args, opts = {}) => runProcess(options.adb, args, { timeout: opts.timeout ?? 600, input: opts.input }); + +const run = async (manifest) => +{ + const state = await adb(["get-state"], { timeout: 30 }); + if (state.code !== 0 || state.stdout.trim() !== "device") { console.error(`no device: adb get-state says ${(state.stdout + state.stderr).trim()}`); process.exit(2); } + + const programs = manifest.programs; + fs.writeFileSync(path.join(options.out, "run.sh"), deviceScript(options.timeout, options.deviceJobs)); + const step = async (what, args, timeout) => + { + const r = await adb(args, { timeout }); + if (r.code !== 0) { console.error(`${what} failed: ${(r.stdout + r.stderr).trim()}`); process.exit(1); } + return r; + }; + await step("clear", ["shell", `rm -rf ${DEVICE} && mkdir -p ${DEVICE}/dl/tests`]); + await step("push run.sh", ["push", path.join(options.out, "run.sh"), `${DEVICE}/run.sh`]); + + // a leg at a time: all of them would not fit on an emulator's data partition + const res = path.join(options.out, "res"); + fs.rmSync(res, { recursive: true, force: true }); + for (const leg of manifest.legs) + { + const mine = programs.map((p, i) => ({ p, i })).filter(({ p }) => p.leg === leg); + if (!mine.length) continue; + fs.writeFileSync(path.join(options.out, "programs.txt"), mine.map(({ p, i }) => `${i} ${p.leg}/${p.name} ${p.cwd} ${p.stdin ? 1 : 0}`).join("\n") + "\n"); + await step(`push ${leg}`, ["push", path.join(options.out, leg), `${DEVICE}/`], 1800); + await step("push programs.txt", ["push", path.join(options.out, "programs.txt"), `${DEVICE}/programs.txt`]); + // pushed from Windows or out of a CI artifact, a program has lost its executable bit + await step("chmod", ["shell", `chmod -R 755 ${DEVICE}/${leg}`]); + + const started = Date.now(); + const ran = await adb(["shell", `sh ${DEVICE}/run.sh`], { timeout: 3 * 3600 }); + console.log(`${leg}: ${mine.length} programs in ${Math.round((Date.now() - started) / 1000)} s, ${options.deviceJobs} at a time: ${(ran.stdout + ran.stderr).trim().split(/\r?\n/).slice(-1)[0]}`); + await step("pull results", ["pull", `${DEVICE}/res`, options.out], 1800); + await step("clean up", ["shell", `rm -rf ${DEVICE}/${leg} ${DEVICE}/res`]); + } + + const suites = makeSuites(options.defaultLibRepo ?? path.resolve(repo, "../TypeScriptCompilerDefaultLib")); + const results = [...manifest.results]; + const read = (f) => (fs.existsSync(f) ? fs.readFileSync(f, "utf8") : ""); + programs.forEach((p, i) => + { + const sum = read(path.join(res, `${i}.sum`)).trim(); + if (!sum) + { + results.push({ leg: p.leg, file: p.file, status: "FAIL", detail: "no result: the device went away, or the run did not reach it" }); + return; + } + + const [code, done, all, errBytes, secs] = sum.split(/\s+/).map(Number); + const out = read(path.join(res, `${i}.otail`)), err = read(path.join(res, `${i}.etail`)); + const detail = lastLines(out + err, 8); + if (code === 124 && secs >= options.timeout - 1) { results.push({ leg: p.leg, file: p.file, status: "TIMEOUT", detail }); return; } + if (code === 124) + { + results.push({ leg: p.leg, file: p.file, status: "FAIL", detail: `ended by a signal after ${secs} s (crash, abort or out of memory)\n${detail}` }); + return; + } + + // what the suite judges, from what came back: the markers as found in the whole output + const suite = p.leg.startsWith("defaultlib-") ? suites.defaultlib : suites.tester; + const failure = suite.passed({ + code, + stdout: out + (done ? "\n[done.]" : "") + (all ? "\n[ALL DONE]" : ""), + stderr: errBytes > 0 && !err.trim() ? "" : err, + }); + results.push(failure ? { leg: p.leg, file: p.file, status: "FAIL", detail: `${failure}${code > 128 ? ` (signal ${code - 128})` : ""}\n${detail}` } + : { leg: p.leg, file: p.file, status: "PASS" }); + }); + + process.exitCode = report(manifest.legs, results, options.out); +}; + +const manifest = options.build ? await build() : JSON.parse(fs.readFileSync(manifestFile, "utf8")); +if (options.run) await run(manifest); +else process.exitCode = manifest.results.some((r) => r.status === "COMPILE") ? 1 : 0; diff --git a/tslang/test/suites.mjs b/tslang/test/suites.mjs new file mode 100644 index 000000000..cbdcca64c --- /dev/null +++ b/tslang/test/suites.mjs @@ -0,0 +1,162 @@ +// What the wasm and Android test runners (wasm/run_tests.mjs, android/run_tests.mjs) share: which +// tests there are, which are skipped, when one passes, and the report. +// +// tester the tester's tests (tslang/test/tester/tests), as test-runner builds them: +// --no-default-lib. A test passes when its program exits with 0 and prints "done.". +// defaultlib the default library's tests (TypeScriptCompilerDefaultLib/tests), linked with the +// default library, as its tests.ps1 runs them, in that repository, with +// tests/.stdin as standard input: a test passes when it prints "ALL DONE", +// writes nothing to stderr and exits with 0. +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +export const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +// the single-file tests ctest runs: `tslang_add_test(NAME test-compile-... COMMAND test-runner [options] "")`. +// The others in the folder are multi-file tests, tests that expect a compile error, or not run at all. +export const testerFiles = () => +{ + const names = new Set(); + const cmake = fs.readFileSync(path.join(repo, "tslang/test/tester/CMakeLists.txt"), "utf8"); + const re = /tslang_add_test\(NAME test-compile-\S+ COMMAND test-runner((?: -[\w=-]+)*) ((?:"[^"]+"\s*)+)\)/g; + let m; + while ((m = re.exec(cmake)) !== null) + { + const testFiles = m[2].match(/"[^"]+"/g); + if (testFiles.length !== 1 || / -shared\b/.test(m[1])) continue; + names.add(testFiles[0].slice(1, -1).replace(/^.*\//, "")); + } + + // a name built in CMake (`${arc_test}.ts`) is not a file + return [...names].filter((name) => fs.existsSync(path.join(repo, "tslang/test/tester/tests", name))); +}; + +// leg: { mm, linux } - linux: the target is Linux (Android is), so "// linux only" tests run +export const makeSuites = (defaultLibRepo) => ({ + tester: { + dir: path.join(repo, "tslang/test/tester/tests"), + files: testerFiles, + compileArgs: ["--no-default-lib"], + passed: (run) => run.code === 0 && (run.stdout + run.stderr).includes("done.") ? null + : run.code !== 0 ? `exit code ${run.code}` : "no 'done.' msg.", + }, + defaultlib: { + dir: path.join(defaultLibRepo, "tests"), + files: () => fs.readdirSync(path.join(defaultLibRepo, "tests")).filter((f) => f.endsWith(".ts")), + compileArgs: ["--nowarn"], + cwd: defaultLibRepo, + marker: (source, leg) => + { + const first = source.split(/\r?\n/, 1)[0]; + if (first.startsWith("// gc only") && leg.mm !== "gc") return "marked '// gc only'"; + if (first.startsWith("// linux only") && !leg.linux) return "marked '// linux only'"; + if (first.startsWith("// not own") && leg.mm === "own") return "marked '// not own'"; + return null; + }, + passed: (run) => run.code !== 0 ? `exit code ${run.code}` + : !run.stdout.includes("ALL DONE") ? "no 'ALL DONE' marker" + : run.stderr.trim() ? "wrote to stderr" : null, + }, +}); + +// a skips file: one test a line, ` [,...] # `. Targets and models +// given limit it to those (wasm32,own: wasm32 under own); none given, every leg. isTarget tells a +// target from a model. +export const loadSkips = (file, isTarget) => +{ + const skips = new Map(); + if (!fs.existsSync(file)) return skips; + for (const line of fs.readFileSync(file, "utf8").split(/\r?\n/)) + { + const m = /^\s*([^#\s]+)(?:\s+([\w,-]+))?\s*(?:#\s*(.*))?$/.exec(line); + if (!m) continue; + const only = m[2] ? m[2].split(",") : []; + skips.set(m[1], { + targets: only.filter(isTarget), + models: only.filter((x) => !isTarget(x)), + reason: m[3] ?? `listed in ${path.basename(file)}`, + }); + } + + return skips; +}; + +// leg: { target, mm, ... } +export const skipReason = (skips, suite, file, source, leg) => +{ + const skip = skips.get(file); + if (skip && (!skip.targets.length || skip.targets.includes(leg.target)) && (!skip.models.length || skip.models.includes(leg.mm))) + { + return skip.reason; + } + + return suite.marker?.(source, leg) ?? null; +}; + +export const runProcess = (command, args, { env, timeout, cwd, input } = {}) => new Promise((resolve) => +{ + const child = spawn(command, args, { env: { ...process.env, ...env }, cwd, stdio: [input === undefined ? "ignore" : "pipe", "pipe", "pipe"] }); + let stdout = "", stderr = "", timedOut = false; + child.stdout.on("data", (d) => (stdout += d)); + child.stderr.on("data", (d) => (stderr += d)); + if (input !== undefined) { child.stdin.on("error", () => { }); child.stdin.end(input); } + const timer = timeout ? setTimeout(() => { timedOut = true; child.kill(); }, timeout * 1000) : null; + child.on("error", (e) => { stderr += String(e); }); + child.on("close", (code) => { if (timer) clearTimeout(timer); resolve({ code, stdout, stderr, timedOut }); }); +}); + +export const lastLines = (text, n) => text.trim().split(/\r?\n/).slice(-n).join("\n"); +export const firstLines = (text, n) => text.trim().split(/\r?\n/).slice(0, n).join("\n"); + +export const FAILED = ["FAIL", "COMPILE", "TIMEOUT"]; + +// results: { leg, file, status, detail }; status PASS, FAIL, COMPILE, TIMEOUT, SKIP, NATIVE (-mm=own +// cannot compile it natively either) or BUILT (compiled, not run). Writes /results.txt, prints +// the summary, and returns the exit code: 1 when anything failed. +export const report = (legNames, results, out) => +{ + results.sort((a, b) => a.leg.localeCompare(b.leg) || a.file.localeCompare(b.file)); + const lines = []; + for (const leg of legNames) + { + const mine = results.filter((r) => r.leg === leg); + const count = (s) => mine.filter((r) => r.status === s).length; + lines.push(`== ${leg}: ${count("PASS")} passed, ${count("FAIL") + count("COMPILE") + count("TIMEOUT")} failed (${count("COMPILE")} to compile, ${count("TIMEOUT")} timed out), ${count("SKIP")} skipped` + + (count("NATIVE") ? `, ${count("NATIVE")} do not compile natively under own either` : "") + + (count("BUILT") ? `, ${count("BUILT")} compiled but not run` : "")); + } + + const text = [...lines, ""]; + for (const r of results.filter((r) => r.status !== "PASS")) + { + text.push(`${r.status} ${r.leg} ${r.file}${r.detail ? "\n " + r.detail.replace(/\n/g, "\n ") : ""}`); + } + + fs.mkdirSync(out, { recursive: true }); + fs.writeFileSync(path.join(out, "results.txt"), text.join("\n") + "\n"); + console.log(lines.join("\n")); + console.log(`details: ${path.join(out, "results.txt")}`); + return results.some((r) => FAILED.includes(r.status)) ? 1 : 0; +}; + +// runs each job through work(job) on `jobs` workers, printing failures and progress +export const runJobs = async (jobs, jobCount, work, results) => +{ + let next = 0, finished = 0; + const total = jobs.length; + const worker = async () => + { + while (next < jobs.length) + { + const result = await work(jobs[next++]); + results.push(result); + finished++; + if (FAILED.includes(result.status)) console.log(`[${finished}/${total}] ${result.status} ${result.leg} ${result.file}`); + else if (finished % 100 === 0) console.log(`[${finished}/${total}] ...`); + } + }; + await Promise.all(Array.from({ length: Math.max(1, jobCount) }, worker)); +}; diff --git a/tslang/test/wasm/run_tests.mjs b/tslang/test/wasm/run_tests.mjs index 4fb08b42e..85e975fe5 100644 --- a/tslang/test/wasm/run_tests.mjs +++ b/tslang/test/wasm/run_tests.mjs @@ -32,15 +32,14 @@ // (--emit=obj): if that fails too, the failure is not wasm's - what -mm=own cannot prove yet - and it // is listed apart (NATIVE), not counted as a failure. ctest runs only a few tests under own natively; // it runs the whole corpus under none and rc, so there a wasm compile error is always a failure. -import { spawn } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import process from "node:process"; import { fileURLToPath } from "node:url"; +import { repo, makeSuites, loadSkips, skipReason, runProcess, lastLines, firstLines, report, runJobs } from "../suites.mjs"; const here = path.dirname(fileURLToPath(import.meta.url)); -const repo = path.resolve(here, "../../.."); const options = { suite: ["tester", "defaultlib"], triple: ["wasm32", "wasm64"], mm: ["none", "rc", "own"], debug: false, @@ -77,94 +76,8 @@ options.tslang ??= [ options.defaultLibRepo ??= path.resolve(repo, "../TypeScriptCompilerDefaultLib"); const defaultLibPath = path.join(options.defaultLibRepo, "__build"); -// wasm_skips.txt: one test a line, ` [,...] # `. Triples and models -// given limit it to those (wasm32,own: wasm32 under own); none given, every leg. -const skips = new Map(); -const skipsFile = path.join(here, "wasm_skips.txt"); -if (fs.existsSync(skipsFile)) -{ - for (const line of fs.readFileSync(skipsFile, "utf8").split(/\r?\n/)) - { - const m = /^\s*([^#\s]+)(?:\s+([\w,]+))?\s*(?:#\s*(.*))?$/.exec(line); - if (!m) continue; - const only = m[2] ? m[2].split(",") : []; - skips.set(m[1], { - triples: only.filter((x) => x.startsWith("wasm")), - models: only.filter((x) => !x.startsWith("wasm")), - reason: m[3] ?? "listed in wasm_skips.txt", - }); - } -} - -// the single-file tests ctest runs: `tslang_add_test(NAME test-compile-... COMMAND test-runner [options] "")` -const testerFiles = () => -{ - const names = new Set(); - const cmake = fs.readFileSync(path.join(repo, "tslang/test/tester/CMakeLists.txt"), "utf8"); - const re = /tslang_add_test\(NAME test-compile-\S+ COMMAND test-runner((?: -[\w=-]+)*) ((?:"[^"]+"\s*)+)\)/g; - let m; - while ((m = re.exec(cmake)) !== null) - { - const testFiles = m[2].match(/"[^"]+"/g); - if (testFiles.length !== 1 || / -shared\b/.test(m[1])) continue; - names.add(testFiles[0].slice(1, -1).replace(/^.*\//, "")); - } - - // a name built in CMake (`${arc_test}.ts`) is not a file - return [...names].filter((name) => fs.existsSync(path.join(repo, "tslang/test/tester/tests", name))); -}; - -const suites = { - tester: { - dir: path.join(repo, "tslang/test/tester/tests"), - files: testerFiles, - compileArgs: ["--no-default-lib"], - passed: (run) => run.code === 0 && (run.stdout + run.stderr).includes("done.") ? null - : run.code !== 0 ? `exit code ${run.code}` : "no 'done.' msg.", - }, - defaultlib: { - dir: path.join(options.defaultLibRepo, "tests"), - files: () => fs.readdirSync(path.join(options.defaultLibRepo, "tests")).filter((f) => f.endsWith(".ts")), - compileArgs: ["--nowarn"], - cwd: options.defaultLibRepo, - marker: (source, leg) => - { - const first = source.split(/\r?\n/, 1)[0]; - if (first.startsWith("// gc only")) return "marked '// gc only'"; - if (first.startsWith("// linux only")) return "marked '// linux only'"; - if (first.startsWith("// not own") && leg.mm === "own") return "marked '// not own'"; - return null; - }, - passed: (run) => run.code !== 0 ? `exit code ${run.code}` - : !run.stdout.includes("ALL DONE") ? "no 'ALL DONE' marker" - : run.stderr.trim() ? "wrote to stderr" : null, - }, -}; - -const skipReason = (suite, file, source, leg) => -{ - const skip = skips.get(file); - if (skip && (!skip.triples.length || skip.triples.includes(leg.triple)) && (!skip.models.length || skip.models.includes(leg.mm))) - { - return skip.reason; - } - - const marked = suite.marker?.(source, leg); - if (marked) return marked; - return null; -}; - -const runProcess = (command, args, { env, timeout, cwd, input } = {}) => new Promise((resolve) => -{ - const child = spawn(command, args, { env: { ...process.env, ...env }, cwd, stdio: [input === undefined ? "ignore" : "pipe", "pipe", "pipe"] }); - let stdout = "", stderr = "", timedOut = false; - child.stdout.on("data", (d) => (stdout += d)); - child.stderr.on("data", (d) => (stderr += d)); - if (input !== undefined) { child.stdin.on("error", () => { }); child.stdin.end(input); } - const timer = timeout ? setTimeout(() => { timedOut = true; child.kill(); }, timeout * 1000) : null; - child.on("error", (e) => { stderr += String(e); }); - child.on("close", (code) => { if (timer) clearTimeout(timer); resolve({ code, stdout, stderr, timedOut }); }); -}); +const skips = loadSkips(path.join(here, "wasm_skips.txt"), (x) => x.startsWith("wasm")); +const suites = makeSuites(options.defaultLibRepo); // the environment the compiler looks the default library up in, and nothing of the user's own lib paths const compileEnv = { DEFAULT_LIB_PATH: defaultLibPath, GC_LIB_PATH: "", LLVM_LIB_PATH: "", TSLANG_LIB_PATH: "" }; @@ -176,7 +89,8 @@ for (const suiteName of options.suite) if (!suites[suiteName]) { console.error(`unknown suite ${suiteName}: tester or defaultlib`); process.exit(2); } for (const triple of options.triple) for (const mm of options.mm) - legs.push({ suiteName, suite: suites[suiteName], triple, mm, name: `${suiteName}-${triple}-${mm}${options.debug ? "-debug" : ""}` }); + legs.push({ suiteName, suite: suites[suiteName], target: triple, triple, mm, linux: false, + name: `${suiteName}-${triple}-${mm}${options.debug ? "-debug" : ""}` }); } const jobs = []; @@ -189,7 +103,7 @@ for (const leg of legs) for (const file of files) { const source = fs.readFileSync(path.join(leg.suite.dir, file), "utf8"); - const reason = skipReason(leg.suite, file, source, leg); + const reason = skipReason(skips, leg.suite, file, source, leg); if (reason) { results.push({ leg: leg.name, file, status: "SKIP", detail: reason }); continue; } jobs.push({ leg, file, outDir }); } @@ -201,7 +115,6 @@ const nodeMajor = Number(process.versions.node.split(".")[0]); const nodeFlags = nodeMajor < 24 ? ["--experimental-wasm-memory64"] : []; const canRun = (leg) => leg.triple !== "wasm64" || nodeMajor >= 24; const runner = path.join(here, "run.mjs"); -const lastLines = (text, n) => text.trim().split(/\r?\n/).slice(-n).join("\n"); const runOne = async ({ leg, file, outDir }) => { @@ -212,7 +125,7 @@ const runOne = async ({ leg, file, outDir }) => ...buildArgs, ...leg.suite.compileArgs, `-o=${wasm}`, source], { env: compileEnv, timeout: 300 }); if (compiled.code !== 0 || !fs.existsSync(wasm)) { - let detail = (compiled.stderr + compiled.stdout).trim().split(/\r?\n/).slice(0, 6).join("\n"); + const detail = firstLines(compiled.stderr + compiled.stdout, 6); if (leg.mm !== "own") return { leg: leg.name, file, status: "COMPILE", detail }; const obj = path.join(outDir, file.replace(/\.ts$/, ".native.o")); @@ -220,7 +133,7 @@ const runOne = async ({ leg, file, outDir }) => `-o=${obj}`, source], { env: compileEnv, timeout: 300 }); fs.rmSync(obj, { force: true }); if (native.code === 0) return { leg: leg.name, file, status: "COMPILE", detail }; - return { leg: leg.name, file, status: "NATIVE", detail: "native: " + (native.stderr + native.stdout).trim().split(/\r?\n/).slice(0, 4).join("\n") }; + return { leg: leg.name, file, status: "NATIVE", detail: "native: " + firstLines(native.stderr + native.stdout, 4) }; } if (!canRun(leg)) @@ -239,43 +152,6 @@ const runOne = async ({ leg, file, outDir }) => return { leg: leg.name, file, status: "PASS" }; }; -let next = 0, finished = 0; -const total = jobs.length; -const worker = async () => -{ - while (next < jobs.length) - { - const job = jobs[next++]; - const result = await runOne(job); - results.push(result); - finished++; - if (["FAIL", "COMPILE", "TIMEOUT"].includes(result.status)) console.log(`[${finished}/${total}] ${result.status} ${result.leg} ${result.file}`); - else if (finished % 100 === 0) console.log(`[${finished}/${total}] ...`); - } -}; - -console.log(`tslang: ${options.tslang}\ndefault lib: ${defaultLibPath}\nlegs: ${legs.map((l) => l.name).join(", ")}; ${total} runs, ${results.length} skips`); -await Promise.all(Array.from({ length: Math.max(1, options.jobs) }, worker)); - -results.sort((a, b) => a.leg.localeCompare(b.leg) || a.file.localeCompare(b.file)); -const lines = []; -for (const leg of legs) -{ - const mine = results.filter((r) => r.leg === leg.name); - const count = (s) => mine.filter((r) => r.status === s).length; - lines.push(`== ${leg.name}: ${count("PASS")} passed, ${count("FAIL") + count("COMPILE") + count("TIMEOUT")} failed (${count("COMPILE")} to compile, ${count("TIMEOUT")} timed out), ${count("SKIP")} skipped` - + (count("NATIVE") ? `, ${count("NATIVE")} do not compile natively under own either` : "") - + (count("BUILT") ? `, ${count("BUILT")} compiled but not run` : "")); -} - -const report = [...lines, ""]; -for (const r of results.filter((r) => r.status !== "PASS")) -{ - report.push(`${r.status} ${r.leg} ${r.file}${r.detail ? "\n " + r.detail.replace(/\n/g, "\n ") : ""}`); -} - -fs.mkdirSync(options.out, { recursive: true }); -fs.writeFileSync(path.join(options.out, "results.txt"), report.join("\n") + "\n"); -console.log(lines.join("\n")); -console.log(`details: ${path.join(options.out, "results.txt")}`); -process.exitCode = results.some((r) => ["FAIL", "COMPILE", "TIMEOUT"].includes(r.status)) ? 1 : 0; +console.log(`tslang: ${options.tslang}\ndefault lib: ${defaultLibPath}\nlegs: ${legs.map((l) => l.name).join(", ")}; ${jobs.length} runs, ${results.length} skips`); +await runJobs(jobs, options.jobs, runOne, results); +process.exitCode = report(legs.map((l) => l.name), results, options.out);