From edaa9425856bc81d6ea62d2685cbb71df5cd3dee Mon Sep 17 00:00:00 2001 From: Hanlu Li Date: Sun, 6 Sep 2026 17:00:17 +0800 Subject: [PATCH 1/5] linux-user, refactor: Share CPU state cloning setup Extract CPU clone initialization without changing existing callers or thread policy. Signed-off-by: Hanlu Li --- linux-user/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/linux-user/main.c b/linux-user/main.c index d01f3a58dc..be8889bb44 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -314,10 +314,9 @@ void init_task_state(TaskState *ts) #endif } -CPUArchState *cpu_copy(CPUArchState *env) +static CPUArchState *cpu_copy_into(CPUArchState *env, CPUState *new_cpu) { CPUState *cpu = env_cpu(env); - CPUState *new_cpu = cpu_create(cpu_type); CPUArchState *new_env = new_cpu->env_ptr; CPUBreakpoint *bp; CPUWatchpoint *wp; @@ -360,6 +359,11 @@ CPUArchState *cpu_copy(CPUArchState *env) return new_env; } +CPUArchState *cpu_copy(CPUArchState *env) +{ + return cpu_copy_into(env, cpu_create(cpu_type)); +} + #if defined(CONFIG_LATX_DEBUG) || defined(CONFIG_DEBUG_TCG) #ifdef CONFIG_LATX #include "latx-options.h" From 182f1cba1df8113bd2bbc5dd57cea4628e2a94e0 Mon Sep 17 00:00:00 2001 From: Hanlu Li Date: Sun, 6 Sep 2026 17:00:18 +0800 Subject: [PATCH 2/5] LATX, refactor: Share Guest callback frames and classify helper calls Preserve the existing callback ABI, including float results and Host floating-point register preservation. Add explicit helper-call entry points for subsequent TLS initialization. Signed-off-by: Hanlu Li --- target/i386/latx/context/callback.c | 101 +++++++++++++++++++++++++--- target/i386/latx/include/callback.h | 2 + 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/target/i386/latx/context/callback.c b/target/i386/latx/context/callback.c index e3fe1150d9..1052bbb74a 100644 --- a/target/i386/latx/context/callback.c +++ b/target/i386/latx/context/callback.c @@ -6,11 +6,17 @@ * SPDX-License-Identifier: MIT */ +#include "config-host.h" + #include +#include +#include +#include #include "callback-args.h" #include "callback-fpr.h" #include "callback.h" +#include "debug.h" #include "lsenv.h" #include "qemu.h" @@ -22,6 +28,14 @@ typedef struct CallbackFrame { size_t stack_words; } CallbackFrame; +typedef struct CallbackResult { + uint64_t rax; + uint64_t rdx; + uint64_t xmm0; + uint64_t xmm1; + unsigned __int128 st0; +} CallbackResult; + static const int callback_gpr_regs[LATX_CALLBACK_GPR_ARGS] = { R_EDI, R_ESI, R_EDX, R_ECX, R_R8, R_R9, }; @@ -75,7 +89,9 @@ static CallbackFrame callback_frame_enter(size_t stack_args, return frame; } -static uint64_t callback_frame_run(CallbackFrame *frame, uintptr_t fnc) +static uint64_t callback_frame_run_result(CallbackFrame *frame, + uintptr_t fnc, + CallbackResult *result) { CPUX86State *cpu = frame->cpu; CPUState *cs = frame->cs; @@ -95,6 +111,17 @@ static uint64_t callback_frame_run(CallbackFrame *frame, uintptr_t fnc) memcpy(&cs->jmp_env, &buf, sizeof(buf)); cpu->eip = oldip; + if (result) { + result->rax = cpu->regs[R_EAX]; + result->rdx = cpu->regs[R_EDX]; + result->xmm0 = cpu->xmm_regs[0].ZMM_Q(0); + result->xmm1 = cpu->xmm_regs[1].ZMM_Q(0); + result->st0 = 0; + memcpy(&result->st0, + &cpu->fpregs[(cpu->fpstt) & 7].d.low, + sizeof(uint64_t)); + } + cpu->regs[R_ESP] += frame->stack_words * sizeof(uint64_t); cpu->regs[R_R15] = Pop64(cpu); cpu->regs[R_R14] = Pop64(cpu); @@ -113,17 +140,31 @@ static uint64_t callback_frame_run(CallbackFrame *frame, uintptr_t fnc) cpu->regs[R_ESP] = frame->old_rbp; cpu->regs[R_EBP] = Pop64(cpu); - return cpu->regs[R_EAX]; + return result ? result->rax : cpu->regs[R_EAX]; +} + +static uint64_t callback_frame_run(CallbackFrame *frame, uintptr_t fnc) +{ + return callback_frame_run_result(frame, fnc, NULL); } #endif -uint64_t RunFunctionWithState(uintptr_t fnc, int nargs, ...) +typedef enum LatxGuestCallKind { + LATX_GUEST_USER_CALLBACK, + LATX_GUEST_INTERNAL_HELPER, + LATX_GUEST_INTERNAL_NO_REFRESH, +} LatxGuestCallKind; + +static uint64_t run_function_with_state_va(uintptr_t fnc, int nargs, + LatxGuestCallKind kind, + va_list *ap) { #ifdef TARGET_X86_64 size_t stack_args; CallbackFrame frame; uint64_t *stack; - va_list ap; + + lsassert(fnc); lsassert(nargs >= 0); @@ -135,23 +176,63 @@ uint64_t RunFunctionWithState(uintptr_t fnc, int nargs, ...) frame = callback_frame_enter(stack_args, false); stack = (uint64_t *)frame.cpu->regs[R_ESP]; - va_start(ap, nargs); for (int i = 0; i < nargs; i++) { if (i < LATX_CALLBACK_GPR_ARGS) { frame.cpu->regs[callback_gpr_regs[i]] = - va_arg(ap, uint64_t); + va_arg(*ap, uint64_t); } else { - *stack++ = va_arg(ap, uint64_t); + *stack++ = va_arg(*ap, uint64_t); } } - va_end(ap); + (void)kind; return callback_frame_run(&frame, fnc); #else + (void)fnc; + (void)nargs; + (void)kind; + (void)ap; return 0; #endif } +uint64_t RunFunctionWithState(uintptr_t fnc, int nargs, ...) +{ + uint64_t result; + va_list ap; + + va_start(ap, nargs); + result = run_function_with_state_va( + fnc, nargs, LATX_GUEST_USER_CALLBACK, &ap); + va_end(ap); + return result; +} + +uint64_t RunFunctionWithStateInternal(uintptr_t fnc, int nargs, ...) +{ + uint64_t result; + va_list ap; + + va_start(ap, nargs); + result = run_function_with_state_va( + fnc, nargs, LATX_GUEST_INTERNAL_HELPER, &ap); + va_end(ap); + return result; +} + +uint64_t RunFunctionWithStateInternalNoRefresh(uintptr_t fnc, int nargs, + ...) +{ + uint64_t result; + va_list ap; + + va_start(ap, nargs); + result = run_function_with_state_va( + fnc, nargs, LATX_GUEST_INTERNAL_NO_REFRESH, &ap); + va_end(ap); + return result; +} + uint64_t RunFunctionFmt(uintptr_t fnc, const char *fmt, ...) { #ifdef TARGET_X86_64 @@ -160,6 +241,8 @@ uint64_t RunFunctionFmt(uintptr_t fnc, const char *fmt, ...) LatxCallbackArgs args; va_list ap; + + lsassert(fnc); lsassert(fmt); lsassert(CODEIS64); @@ -200,6 +283,8 @@ float RunFunctionFmtFloat(uintptr_t fnc, const char *fmt, ...) LatxCallbackArgs args; va_list ap; + + lsassert(fnc); lsassert(fmt); lsassert(CODEIS64); diff --git a/target/i386/latx/include/callback.h b/target/i386/latx/include/callback.h index 76e4aac680..cf1b5a9daf 100644 --- a/target/i386/latx/include/callback.h +++ b/target/i386/latx/include/callback.h @@ -6,6 +6,8 @@ uint64_t RunFunctionWithState(uintptr_t fnc, int nargs, ...); uint64_t RunFunctionFmt(uintptr_t fnc, const char *fmt, ...); float RunFunctionFmtFloat(uintptr_t fnc, const char *fmt, ...); +uint64_t RunFunctionWithStateInternal(uintptr_t fnc, int nargs, ...); +uint64_t RunFunctionWithStateInternalNoRefresh(uintptr_t fnc, int nargs, ...); #define RunFunction RunFunctionWithState #endif //__CALLBACK_H__ From e183e0fbb8413602d9ede519d8e74de64cc9cd07 Mon Sep 17 00:00:00 2001 From: Hanlu Li Date: Sun, 6 Sep 2026 17:00:18 +0800 Subject: [PATCH 3/5] linux-user, fix: Preserve the selected Guest runtime root across exec Keep common runtime options available in release builds and propagate an explicit -L selection through LAT_LD_PREFIX. A re-executed Guest must use the same loader and libc as its parent. Add test-runtime-prefix-exec to check the exported selection and mapped Guest libc before and after exec. Reverting the runtime-root fix makes the fixture fail; the fixed build passes on LoongArch ABI1. Signed-off-by: Hanlu Li --- linux-user/main.c | 10 +++- .../registrations/process/meson.build | 9 ++++ tests/integration/runtime-prefix-exec.c | 47 +++++++++++++++++++ tests/integration/test-runtime-prefix-exec.sh | 28 +++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 tests/integration/runtime-prefix-exec.c create mode 100755 tests/integration/test-runtime-prefix-exec.sh diff --git a/linux-user/main.c b/linux-user/main.c index be8889bb44..6de768c190 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -430,6 +430,7 @@ static void handle_arg_latx_disassemble_trace_cmp(const char *arg) } #endif +#endif /* CONFIG_LATX_DEBUG || CONFIG_DEBUG_TCG */ static void handle_arg_imm_skip_pc(const char *arg) { imm_skip_pc = strtol(arg, NULL, 16); @@ -606,7 +607,6 @@ static void handle_arg_plugin(const char *arg) qemu_plugin_opt_parse(arg, &plugins); } #endif -#endif static void handle_arg_help(const char *arg) { @@ -629,6 +629,12 @@ static void handle_arg_runtime_info(const char *arg) static void handle_arg_ld_prefix(const char *arg) { + g_autofree char *setting = g_strdup_printf( + "LAT_LD_PREFIX=%s", arg); + + if (!setting || envlist_setenv(envlist, setting) != 0) { + usage(EXIT_FAILURE); + } interp_prefix = strdup(arg); latx_runtime_prefix_selected(); } @@ -1062,6 +1068,7 @@ static const struct qemu_argument arg_table[] = { true, handle_arg_latx_disassemble_trace_cmp, "", "LATX Compare different disassemble."}, #endif +#endif /* CONFIG_LATX_DEBUG || CONFIG_DEBUG_TCG */ {"g", "LAT_GDB", true, handle_arg_gdb, "port", "wait gdb connection to 'port'"}, {"s", "LAT_STACK_SIZE", true, handle_arg_stack_size, @@ -1104,7 +1111,6 @@ static const struct qemu_argument arg_table[] = { #ifdef CONFIG_PLUGIN {"plugin", "LAT_PLUGIN", true, handle_arg_plugin, "", "[file=][,arg=]"}, -#endif #endif {"h", NULL, false, handle_arg_help, "", "print this help"}, diff --git a/tests/integration/registrations/process/meson.build b/tests/integration/registrations/process/meson.build index f1fd68afbd..6ffe7dcd41 100644 --- a/tests/integration/registrations/process/meson.build +++ b/tests/integration/registrations/process/meson.build @@ -8,6 +8,15 @@ if 'x86_64-linux-user' in target_dirs ], 'timeout': 120, }] + latx_integration_tests += [{ + 'name': 'test-runtime-prefix-exec', + 'runner': find_program('../../test-runtime-prefix-exec.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../runtime-prefix-exec.c'), + ], + 'timeout': 60, + }] latx_integration_tests += [{ 'name': 'test-proc-readdir', 'runner': find_program('../../test-proc-readdir.sh'), diff --git a/tests/integration/runtime-prefix-exec.c b/tests/integration/runtime-prefix-exec.c new file mode 100644 index 0000000000..31f5bcb548 --- /dev/null +++ b/tests/integration/runtime-prefix-exec.c @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#define _GNU_SOURCE +#include +#include +#include +#include + +int main(int argc, char **argv) +{ + const char *prefix = getenv("LAT_LD_PREFIX"); + char *line = NULL; + size_t capacity = 0; + int mapped = 0; + FILE *maps; + + if (argc != 3 || !prefix || strcmp(prefix, argv[1])) { + fprintf(stderr, "FAIL: explicit -L was not exported to the Guest\n"); + return 1; + } + maps = fopen("/proc/self/maps", "r"); + if (!maps) { + return 2; + } + while (getline(&line, &capacity, maps) >= 0) { + if (strstr(line, argv[1]) && strstr(line, "libc.so.6")) { + mapped = 1; + } + } + free(line); + fclose(maps); + if (!mapped) { + fprintf(stderr, "FAIL: libc did not come from the selected root\n"); + return 3; + } + if (!strcmp(argv[2], "parent")) { + char *child_argv[] = { argv[0], argv[1], "child", NULL }; + + execve(argv[0], child_argv, environ); + perror("execve"); + return 4; + } + if (strcmp(argv[2], "child")) { + return 5; + } + puts("PASS: explicit runtime prefix and mapped libc survive exec"); + return 0; +} diff --git a/tests/integration/test-runtime-prefix-exec.sh b/tests/integration/test-runtime-prefix-exec.sh new file mode 100755 index 0000000000..72c14c17ac --- /dev/null +++ b/tests/integration/test-runtime-prefix-exec.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later +set -eu +emulator=$1 +source_file=$2 +guest_root=${LATX_X86_64_SYSROOT:-/usr/gnemul/latx-x86_64} +guest_cc=${LATX_X86_64_CC:-x86_64-linux-gnu-gcc} +if ! command -v "$guest_cc" >/dev/null 2>&1; then + echo 'SKIP: requires a Guest C compiler' + exit 77 +fi +if [ ! -r "$guest_root/lib64/ld-linux-x86-64.so.2" ] || + [ ! -r "$guest_root/lib/x86_64-linux-gnu/libc.so.6" ]; then + echo 'SKIP: requires a matching x86-64 glibc runtime' + exit 77 +fi +task_dir=$(mktemp -d) +trap 'rm -rf "$task_dir"' EXIT HUP INT TERM +selected="$task_dir/selected-runtime" +mkdir -p "$selected/lib64" "$selected/lib/x86_64-linux-gnu" +cp -L "$guest_root/lib64/ld-linux-x86-64.so.2" "$selected/lib64/" +cp -L "$guest_root/lib/x86_64-linux-gnu/libc.so.6" \ + "$selected/lib/x86_64-linux-gnu/" +"$guest_cc" --sysroot="$guest_root" -O2 -Wall -Wextra -Werror \ + "$source_file" -o "$task_dir/runtime-prefix-exec" +LATX_AOT=0 LATX_KZT=0 LAT_LD_PREFIX="$guest_root" \ + "$emulator" -L "$selected" "$task_dir/runtime-prefix-exec" \ + "$selected" parent From a1991d92a062cfd4af09bbc9fe89bb0a3508f8f1 Mon Sep 17 00:00:00 2001 From: Hanlu Li Date: Sun, 6 Sep 2026 17:00:25 +0800 Subject: [PATCH 4/5] LATX, feat: Add opt-in Guest TLS for native-thread callbacks Keep Guest TLS management disabled unless LATX_KZT_GUEST_TLS=1 and KZT is effective. Attach native-created threads from an immutable template, initialize Guest TLS from live loader state, and coordinate generation refresh, callback execution, fork and thread exit. Keep existing library registration and disabled-mode loader behavior. Add policy, loader and native-thread callback tests. Initialize the attached TCB thread ID and robust-list fields before calling the Guest loader allocation helper. The debugger regression reaches that helper with TID 0 before the fix and with the real thread ID after it. Opt-in off/on, robust owner death and 64-cycle dynamic loading regressions pass on LoongArch ABI1. Supply the new runtime-gate variables in the standalone TB-flush test fixture so the complete KZT-enabled lat-pr-fast suite links and runs. Co-authored-by: sunguoyun Signed-off-by: Hanlu Li --- accel/tcg/tb-flush.c | 13 +- docs/devel/kzt-guest-tls.md | 146 + linux-user/main.c | 34 + linux-user/qemu.h | 7 + linux-user/syscall.c | 473 +++- target/i386/cpu.h | 6 + target/i386/latx/context/callback.c | 216 +- target/i386/latx/context/elfloader.c | 37 + target/i386/latx/context/kzt-guest-thread.c | 501 ++++ target/i386/latx/context/kzt-guest-tls.c | 2401 +++++++++++++++++ .../latx/context/kzt_public_loader_observer.c | 1444 +++++++++- target/i386/latx/context/meson.build | 3 + target/i386/latx/context/myalign.c | 871 +++++- target/i386/latx/context/wrappedlibc.c | 39 +- target/i386/latx/context/wrappedlibdl.c | 511 +++- target/i386/latx/context/wrappedlibx11.c | 9 +- target/i386/latx/context/wrappedlibxcb.c | 28 +- target/i386/latx/context/x86dlfun.c | 16 + target/i386/latx/include/callback.h | 16 +- target/i386/latx/include/elfloader.h | 1 + target/i386/latx/include/kzt-guest-thread.h | 30 + .../i386/latx/include/kzt-guest-tls-epoch.h | 48 + target/i386/latx/include/kzt-guest-tls.h | 203 ++ target/i386/latx/include/kzt-runtime.h | 7 + .../latx/include/kzt_public_loader_observer.h | 82 + target/i386/latx/include/latx-options.h | 2 + target/i386/latx/include/myalign.h | 26 + .../i386/latx/include/wrappedlibc_private.h | 11 +- target/i386/latx/latx-options.c | 9 + .../latx/sbt/tests/aot-cache-reader-test.c | 1 + .../latx/sbt/tests/aot-file-publish-test.c | 1 + .../latx/sbt/tests/aot-merge-memory-test.c | 1 + .../tests/tb-flush-smc-reload-async-test.c | 2 + tests/integration/kzt-attach-busy.gdb | 49 + tests/integration/kzt-attached-robust-guest.c | 130 + tests/integration/kzt-attached-robust-probe.c | 97 + tests/integration/kzt-bootstrap-tid.gdb | 31 + .../kzt-cxx-tls-lifetime-destructor.cpp | 17 + .../integration/kzt-cxx-tls-lifetime-guest.c | 126 + .../kzt-cxx-tls-lifetime-plugin.cpp | 27 + .../integration/kzt-cxx-tls-lifetime-probe.c | 77 + .../integration/kzt-cxx-tls-lifetime-shared.h | 16 + .../integration/kzt-guest-tls-opt-in-plugin.c | 7 + tests/integration/kzt-guest-tls-opt-in.c | 273 ++ .../kzt-host-thread-cxx-tls-plugin.cpp | 43 + .../kzt-host-thread-tls-guest-plugin-b.c | 25 + .../kzt-host-thread-tls-guest-plugin.c | 54 + .../kzt-host-thread-tls-ie-plugin.c | 57 + .../kzt-host-thread-tls-tlsdesc-plugin.c | 20 + .../integration/kzt-pthread-tsd-alias-guest.c | 216 ++ tests/integration/kzt-pthread-tsd-alias.h | 21 + .../integration/kzt-tls-dlopen-stress-guest.c | 553 ++++ .../kzt-tls-dlopen-stress-ie-plugin.c | 68 + .../integration/kzt-tls-dlopen-stress-probe.c | 324 +++ .../kzt-tls-dlopen-stress-shared.h | 8 + .../kzt-tls-fork-lifecycle-guest.c | 206 ++ .../kzt-tls-fork-lifecycle-plugin.c | 4 + .../kzt-tls-fork-lifecycle-probe.c | 109 + .../registrations/x11-kzt/meson.build | 83 + tests/integration/test-kzt-attached-robust.sh | 99 + .../integration/test-kzt-cxx-tls-lifetime.sh | 138 + .../integration/test-kzt-guest-tls-opt-in.sh | 62 + .../integration/test-kzt-pthread-tsd-alias.sh | 87 + .../integration/test-kzt-tls-dlopen-stress.sh | 149 + .../test-kzt-tls-fork-lifecycle.sh | 94 + tests/integration/x11-async-bridge-dummy.c | 16 + .../kzt/check_kzt_tls_refresh_boundary.py | 32 + tests/unit/kzt/test_kzt_guest_tls_epoch.c | 59 + tests/unit/kzt/test_kzt_guest_tls_policy.c | 38 + .../kzt/test_kzt_public_loader_observer.c | 931 ++++++- tests/unit/meson.build | 35 + 71 files changed, 11330 insertions(+), 246 deletions(-) create mode 100644 docs/devel/kzt-guest-tls.md create mode 100644 target/i386/latx/context/kzt-guest-thread.c create mode 100644 target/i386/latx/context/kzt-guest-tls.c create mode 100644 target/i386/latx/include/kzt-guest-thread.h create mode 100644 target/i386/latx/include/kzt-guest-tls-epoch.h create mode 100644 target/i386/latx/include/kzt-guest-tls.h create mode 100644 tests/integration/kzt-attach-busy.gdb create mode 100644 tests/integration/kzt-attached-robust-guest.c create mode 100644 tests/integration/kzt-attached-robust-probe.c create mode 100644 tests/integration/kzt-bootstrap-tid.gdb create mode 100644 tests/integration/kzt-cxx-tls-lifetime-destructor.cpp create mode 100644 tests/integration/kzt-cxx-tls-lifetime-guest.c create mode 100644 tests/integration/kzt-cxx-tls-lifetime-plugin.cpp create mode 100644 tests/integration/kzt-cxx-tls-lifetime-probe.c create mode 100644 tests/integration/kzt-cxx-tls-lifetime-shared.h create mode 100644 tests/integration/kzt-guest-tls-opt-in-plugin.c create mode 100644 tests/integration/kzt-guest-tls-opt-in.c create mode 100644 tests/integration/kzt-host-thread-cxx-tls-plugin.cpp create mode 100644 tests/integration/kzt-host-thread-tls-guest-plugin-b.c create mode 100644 tests/integration/kzt-host-thread-tls-guest-plugin.c create mode 100644 tests/integration/kzt-host-thread-tls-ie-plugin.c create mode 100644 tests/integration/kzt-host-thread-tls-tlsdesc-plugin.c create mode 100644 tests/integration/kzt-pthread-tsd-alias-guest.c create mode 100644 tests/integration/kzt-pthread-tsd-alias.h create mode 100644 tests/integration/kzt-tls-dlopen-stress-guest.c create mode 100644 tests/integration/kzt-tls-dlopen-stress-ie-plugin.c create mode 100644 tests/integration/kzt-tls-dlopen-stress-probe.c create mode 100644 tests/integration/kzt-tls-dlopen-stress-shared.h create mode 100644 tests/integration/kzt-tls-fork-lifecycle-guest.c create mode 100644 tests/integration/kzt-tls-fork-lifecycle-plugin.c create mode 100644 tests/integration/kzt-tls-fork-lifecycle-probe.c create mode 100755 tests/integration/test-kzt-attached-robust.sh create mode 100755 tests/integration/test-kzt-cxx-tls-lifetime.sh create mode 100755 tests/integration/test-kzt-guest-tls-opt-in.sh create mode 100755 tests/integration/test-kzt-pthread-tsd-alias.sh create mode 100755 tests/integration/test-kzt-tls-dlopen-stress.sh create mode 100755 tests/integration/test-kzt-tls-fork-lifecycle.sh create mode 100644 tests/unit/kzt/check_kzt_tls_refresh_boundary.py create mode 100644 tests/unit/kzt/test_kzt_guest_tls_epoch.c create mode 100644 tests/unit/kzt/test_kzt_guest_tls_policy.c diff --git a/accel/tcg/tb-flush.c b/accel/tcg/tb-flush.c index 32c9a3ba6b..6cf0a3fede 100644 --- a/accel/tcg/tb-flush.c +++ b/accel/tcg/tb-flush.c @@ -108,10 +108,15 @@ void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) qemu_plugin_flush_cb(); } #if defined(CONFIG_LATX_KZT) - CPU_FOREACH(cpu) { - /* The installer also checks the effective library-group mask. */ - if (cpu && option_kzt) { - kzt_install_runtime_callbacks(cpu, &info1); + if (!latx_kzt_guest_tls_enabled() || did_flush) { + CPU_FOREACH(cpu) { + /* The installer checks the effective library-group mask. */ + if (cpu && option_kzt) { + kzt_install_runtime_callbacks(cpu, &info1); + if (latx_kzt_guest_tls_enabled()) { + break; + } + } } } #endif diff --git a/docs/devel/kzt-guest-tls.md b/docs/devel/kzt-guest-tls.md new file mode 100644 index 0000000000..0337d64997 --- /dev/null +++ b/docs/devel/kzt-guest-tls.md @@ -0,0 +1,146 @@ +# Optional Guest TLS for KZT native-thread callbacks + +KZT can receive Guest callbacks on pthreads created by a native library. +Such a thread has not executed the Guest pthread creation path. The optional +Guest TLS runtime provides a Guest CPU, stack, TCB and DTV for that thread. + +## Activation + +The feature is disabled by default. Select it at process startup: + +```sh +LATX_KZT=1 LATX_KZT_GUEST_TLS=1 latx-x86_64 program +``` + +The equivalent command-line option is `-latx-kzt-guest-tls 1`. +Only `0` and `1` are accepted. Enabling this option without an effective +KZT library group does not activate the feature. Configuration is fixed +before Guest execution and must not be changed after threads are attached. + +With the option disabled, no Host-thread template is created, automatic +attachment is disabled, and the added loader transaction and TLS refresh +locks are bypassed. Existing library registration remains available. +Guest pthreads continue to own their Guest libc TLS. + +## Ownership and lifetime + +An immutable template is captured at the Guest program entry point. An +unattached Host thread copies this template, creates its own Guest stack +and constructs Guest TLS from validated live loader information. It never +copies a concurrently executing parent's CPU state or entire pthread +descriptor. + +The loader observer reads Guest r_debug/link_map and ELF metadata. Static +TLS placement, dynamic module IDs, object identity and DTV generation must +agree before Guest code executes. An unchanged loader epoch allows a +callback to reuse its TLS without another link_map walk. + +Callback execution protects the attached thread's DTV from concurrent +replacement. Loader changes and fork coordinate with this protection. +Internal initialization calls use an explicit no-refresh entry point to +avoid recursively entering the initializer. + +Guest pthread keys and values stay in Guest libc. Optional key wrappers +record destructors while forwarding key operations to Guest libc. Attached +thread exit runs recorded TSD and C++ TLS destructors, releases retained +DSOs and handles non-PI robust mutex owner death before releasing TLS. +The standard Guest pthread exit path retains its own destructor ownership. + +If Guest code running on an attached thread creates a Guest pthread, CPU +cloning clears the managed stack, TLS allocation, parent snapshot and +destructor-state pointers. Guest libc supplies the child's own TLS through +the normal clone path. The child must not inherit the attached parent's +DTV ownership or execution lock. + +## Scope + +This feature does not enable bidirectional errno or locale synchronization. +The separate libc boundary facility requires explicit activation. +Constructing a usable Guest libc thread state still initializes that +thread's Guest locale/ctype data. + +The implementation targets x86-64 glibc Guest TLS. Other libc layouts, +additional loader namespaces, PI robust futexes and arbitrary non-local +exits across a native callback require separate validation. + +This does not implement all private glibc pthread initialization. In +particular, the extended ABI1 resolver probe currently observes shared +`__res_state()` backing storage on attached threads. Resolver APIs are +outside the supported attached-thread profile until that bootstrap is +implemented and validated. + +The Guest loader must expose the TLS allocation helpers and a loaded +`dlinfo` provider for module IDs that cannot be established from relocation +evidence. On systems with a separate Guest libdl, the caller must load/link +that library before attachment. Missing helpers reject attachment; the +runtime does not infer private link_map offsets or guess module IDs. + +## Tests + +`test-kzt-guest-tls-policy` exercises the opt-in policy. +`test-kzt-public-loader-observer` exercises live ELF validation. +`test-kzt-guest-tls-epoch` exercises snapshot reuse. +`test-kzt-guest-tls-opt-in` uses a native probe library to check existing +thread callbacks with the option unset/zero and isolated Host-thread TLS +with the option enabled, including Guest TSD destruction. + +With a debug-symbol LATX build, set `LATX_KZT_BOOTSTRAP_GDB=gdb` when +running `test-kzt-guest-tls-opt-in.sh` to additionally check the actual +Guest TLS allocation helper entry. The diagnostic checks that every +observed attached TCB already contains its real thread ID before the +loader helper executes, and fails if that entry is never reached. +The second diagnostic injects one transient busy loader snapshot before +Guest TLS allocation and verifies that attachment retries and executes the +callback. Attachment retries release the failed attempt's resources and +locks before waiting; other initialization failures are not retried. + +These test binaries are not part of the default product build. +Target tests require a LoongArch host and matching Guest compilation tools. +SKIP is not a successful runtime result. + + +## Inventory and fork lifetime + +The KZT inventory version is a private observation counter. It is never used +as the Guest DTV generation. A separate loader generation is seeded from a DTV +initialized by the Guest loader. The `r_brk` observer records every Guest-loader +transaction whose TLS inventory changes and advances this generation once per +transaction, so several load and unload operations cannot collapse into one +attached-thread observation. + +The early `r_brk` callback may run before glibc publishes the new loader +generation. It prepares new TLS entries for constructors but leaves the +attached inventory dirty. A callback reached while the serialized Guest loader +operation is active uses the same preparation-only path. The regular +post-`dlopen` refresh publishes the separately tracked loader generation without +reinitializing live TLS pointers or values. + +KZT-only external registrations are tracked in the inventory but do not advance +the Guest loader's TLS generation. They therefore propagate their prepared DTV +entries without advancing the loader generation. + +Ordinary attached callbacks may suspend their execution guard at selected +blocking I/O and non-PI futex wait syscalls. Their CallbackScope retains the +CPU/TLS allocation and keeps native cancellation disabled. Internal helper +calls and a thread already preparing fork do not take this suspension path. +Syscalls changing mappings, TLS or thread identity retain their existing +serialization. + +A Guest libc fork hook takes the fork writer lock before libc takes its +internal locks. Its Guest address survives a full TB flush, but the saved +translated instructions do not: both the loader and fork hooks are rebuilt. +A new executable clears the cached addresses. The libc-scoped lookup avoids +ambiguity when libpthread also exports fork. + +Fork completion releases an outstanding early writer even when seccomp or +argument validation bypasses do_fork. A real internal syscall restart keeps +the same fork scope; a seccomp errno numerically equal to a restart code does +not. Cleanup is idempotent when do_fork already finished it. + +The test-kzt-tls-fork-* integration cases cover retained GD TLS values after +coalesced loader activity, observed blocking read/futex waits, denied fork, +restart-valued errno, denied fork from attached callers, and fork hooks after an explicit full +TB flush with Guest libc lock contention. The flush case uses gdb to observe +hook execution; the other tests require no debugger. Set +LATX_KZT_FORK_GUEST_ARTIFACT_DIR for verified prebuilt Guest fixtures and +LATX_X11_INCLUDE when the Xlib development headers are outside /usr/include. diff --git a/linux-user/main.c b/linux-user/main.c index 6de768c190..4c9080ff96 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -79,6 +79,7 @@ int mydebug = 1; #include "wrapper.h" #if defined(CONFIG_LATX_KZT) #include "kzt-groups.h" +#include "kzt-guest-tls.h" #include "wrappertbbridge.h" box64context_t* my_context = NULL; elfheader_t* elf_header = NULL; @@ -326,6 +327,13 @@ static CPUArchState *cpu_copy_into(CPUArchState *env, CPUState *new_cpu) new_cpu->tcg_cflags = cpu->tcg_cflags; memcpy(new_env, env, sizeof(CPUArchState)); +#ifdef CONFIG_LATX_KZT + /* Managed resources belong to one CPU and must not be inherited. */ + new_env->kzt_guest_stack_base = 0; + new_env->kzt_guest_tls_allocation = NULL; + new_env->kzt_guest_tls_parent_snapshot = NULL; + new_env->kzt_guest_thread_state = NULL; +#endif /* * NOTE: Current QEMU only has one and only one gdt_table ptr. @@ -696,6 +704,26 @@ static void handle_arg_latx_kzt(const char *arg) option_kzt = value; } +static void handle_arg_latx_kzt_guest_tls(const char *arg) +{ + g_clear_pointer(&option_kzt_guest_tls_error, g_free); + if (!strcmp(arg, "0") || !strcmp(arg, "1")) { +#ifndef TARGET_X86_64 + if (arg[0] == '1') { + option_kzt_guest_tls = 0; + option_kzt_guest_tls_error = + g_strdup("LATX_KZT_GUEST_TLS requires an x86-64 Guest"); + return; + } +#endif + option_kzt_guest_tls = arg[0] == '1'; + return; + } + option_kzt_guest_tls = 0; + option_kzt_guest_tls_error = g_strdup_printf( + "LATX_KZT_GUEST_TLS must be exactly 0 or 1 (got '%s')", arg); +} + static void handle_arg_latx_kzt_libs(const char *arg) { g_free(option_kzt_libs); @@ -979,6 +1007,9 @@ static const struct qemu_argument arg_table[] = { #if defined(CONFIG_LATX_KZT) {"latx-kzt", "LATX_KZT", true, handle_arg_latx_kzt, "", "enable kuzhitong"}, + {"latx-kzt-guest-tls", "LATX_KZT_GUEST_TLS", true, + handle_arg_latx_kzt_guest_tls, "0|1", + "Enable Guest TLS for native-thread callbacks (default: 0)"}, {"latx-kzt-libs", "LATX_KZT_LIBS", true, handle_arg_latx_kzt_libs, "group,...", "select KZT library groups"}, {"latx-kzt-log", "LATX_KZT_LOG", true, handle_arg_latx_kzt_log, @@ -1634,6 +1665,9 @@ int main(int argc, char **argv, char **envp) latx_handle_args(exec_path); #endif thread_cpu = cpu; +#ifdef CONFIG_LATX + latx_register_host_thread_template(env); +#endif /* * Reserving too much vm space via mmap can run into problems diff --git a/linux-user/qemu.h b/linux-user/qemu.h index 3b566ce597..363c8fe309 100644 --- a/linux-user/qemu.h +++ b/linux-user/qemu.h @@ -372,8 +372,15 @@ abi_long do_syscall_with_seccomp(void *cpu_env, int num, int seccomp_num, abi_long arg5, abi_long arg6, abi_long arg7, abi_long arg8); extern __thread CPUState *thread_cpu; +#ifdef CONFIG_LATX +void latx_register_host_thread_template(CPUArchState *env); +int latx_finalize_host_thread_template(CPUArchState *env); +int latx_attach_current_host_thread(void); +#endif void cpu_loop(CPUArchState *env); const char *target_strerror(int err); +int host_to_target_errno(int err); +int target_to_host_errno(int err); int get_osversion(void); void init_qemu_uname_release(void); void fork_start(void); diff --git a/linux-user/syscall.c b/linux-user/syscall.c index 24b666f875..827771718b 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -159,6 +159,9 @@ #ifdef CONFIG_LATX #define TUNNEL_VIRTUAL_SYSCALL_ID 600 #include "lsenv.h" +#include "kzt-guest-tls.h" +#include "kzt-guest-thread.h" +#include "myalign.h" #include #include "aot.h" #include "latx-options.h" @@ -1201,7 +1204,7 @@ static uint16_t host_to_target_errno_table[ERRNO_TABLE_SIZE] = { #endif }; -static inline int host_to_target_errno(int err) +int host_to_target_errno(int err) { if (err >= 0 && err < ERRNO_TABLE_SIZE && host_to_target_errno_table[err]) { @@ -1210,7 +1213,7 @@ static inline int host_to_target_errno(int err) return err; } -static inline int target_to_host_errno(int err) +int target_to_host_errno(int err) { if (err >= 0 && err < ERRNO_TABLE_SIZE && target_to_host_errno_table[err]) { @@ -9590,9 +9593,292 @@ abi_long do_arch_prctl(CPUX86State *env, int code, abi_ulong addr) #endif /* defined(TARGET_I386) */ #define NEW_STACK_SIZE 0x200000 +#define LATX_HOST_THREAD_ATTACH_RETRIES 32 static pthread_mutex_t clone_lock = PTHREAD_MUTEX_INITIALIZER; +#ifdef CONFIG_LATX +static CPUArchState *latx_host_thread_template; +static pthread_key_t latx_host_thread_key; +static pthread_once_t latx_host_thread_key_once = PTHREAD_ONCE_INIT; +static int latx_host_thread_key_error; + +static void latx_host_thread_destructor(void *opaque); + +static void latx_host_thread_key_init(void) +{ + latx_host_thread_key_error = + pthread_key_create(&latx_host_thread_key, + latx_host_thread_destructor); +} + +static void latx_host_thread_release_cpu(CPUState *cpu) +{ + CPUX86State *env = cpu->env_ptr; + TaskState *ts = cpu->opaque; +#ifdef CONFIG_LATX_FAST_JMPCACHE + void *fast_jmp_cache = env->tb_jmp_cache_ptr; +#endif + + kzt_guest_tls_destroy(env); +#ifdef CONFIG_LATX_FAST_JMPCACHE + env->tb_jmp_cache_ptr = NULL; +#endif + if (env->gdt.base) { + target_munmap(env->gdt.base, + sizeof(uint64_t) * TARGET_GDT_ENTRIES, 0); + env->gdt.base = 0; + } + if (env->kzt_guest_stack_base) { + target_munmap(env->kzt_guest_stack_base, NEW_STACK_SIZE, 0); + env->kzt_guest_stack_base = 0; + } + object_property_set_bool(OBJECT(cpu), "realized", false, NULL); + object_unparent(OBJECT(cpu)); + object_unref(OBJECT(cpu)); +#ifdef CONFIG_LATX_FAST_JMPCACHE + latx_fast_jmp_cache_free_rcu(fast_jmp_cache); +#endif + g_free(ts); +} + +static void latx_host_thread_destructor(void *opaque) +{ + CPUState *cpu = opaque; + + kzt_guest_thread_destroy(cpu->env_ptr); + if (kzt_guest_tls_cleanup_robust_list(cpu->env_ptr) != 0) { + fprintf(stderr, + "KZT Guest robust-list cleanup failed; " + "refusing to continue\n"); + _exit(EXIT_FAILURE); + } + pthread_mutex_lock(&clone_lock); + latx_host_thread_release_cpu(cpu); + pthread_mutex_unlock(&clone_lock); + + thread_cpu = NULL; + lsenv = NULL; + rcu_unregister_thread(); +} + +void latx_register_host_thread_template(CPUArchState *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + CPUState *cpu = env_cpu(env); + + if (!close_latx_parallel) { + cpu->tcg_cflags |= CF_PARALLEL; + } +} + +int latx_finalize_host_thread_template(CPUArchState *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return 0; + } + CPUState *source_cpu; + CPUState *template_cpu; + TaskState *source_ts; + TaskState *template_ts; + CPUArchState *template_env; + int result = -1; + + if (!env || !env->segs[R_FS].base) { + return -1; + } + pthread_mutex_lock(&clone_lock); + if (latx_host_thread_template) { + template_cpu = env_cpu(latx_host_thread_template); + latx_host_thread_template = NULL; + latx_host_thread_release_cpu(template_cpu); + } + + source_cpu = env_cpu(env); + source_ts = source_cpu ? source_cpu->opaque : NULL; + if (!source_ts) { + goto out; + } + template_env = cpu_copy(env); + if (!template_env) { + goto out; + } + template_cpu = env_cpu(template_env); + cpu_list_remove(template_cpu); + + template_ts = g_new0(TaskState, 1); + init_task_state(template_ts); + template_ts->bprm = source_ts->bprm; + template_ts->info = source_ts->info; + template_ts->signal_mask = source_ts->signal_mask; + template_ts->seccomp_filter = source_ts->seccomp_filter; + template_ts->ipc_namespace_isolated = + source_ts->ipc_namespace_isolated; + template_cpu->opaque = template_ts; +#ifdef CONFIG_LATX_FAST_JMPCACHE + template_env->tb_jmp_cache_ptr = NULL; +#endif +#ifdef CONFIG_LATX_KZT + template_env->kzt_guest_tls_parent_snapshot = NULL; + template_env->kzt_guest_tls_allocation = NULL; + template_env->kzt_guest_thread_state = NULL; + if (kzt_guest_tls_snapshot_parent(env, template_env) != 0) { + latx_host_thread_release_cpu(template_cpu); + goto out; + } +#endif + template_env->kzt_guest_stack_base = 0; + latx_host_thread_template = template_env; + result = 0; + +out: + pthread_mutex_unlock(&clone_lock); + return result; +} + +static int latx_attach_current_host_thread_once(void) +{ + CPUArchState *parent_env; + CPUState *parent_cpu; + TaskState *parent_ts; + CPUArchState *new_env; + CPUState *new_cpu; + TaskState *ts; + abi_long stack; + int old_cancel_state; + int ret; + + if (thread_cpu && lsenv && lsenv->cpu_state) { + return 0; + } + if (!latx_kzt_guest_tls_enabled()) { + return -1; + } + ret = pthread_once(&latx_host_thread_key_once, + latx_host_thread_key_init); + if (ret || latx_host_thread_key_error) { + return -1; + } + ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, + &old_cancel_state); + if (ret) { + return -1; + } + ts = g_new0(TaskState, 1); + init_task_state(ts); + + pthread_mutex_lock(&clone_lock); + parent_env = latx_host_thread_template; + parent_cpu = parent_env ? env_cpu(parent_env) : NULL; + parent_ts = parent_cpu ? (TaskState *)parent_cpu->opaque : NULL; + if (!parent_ts) { + pthread_mutex_unlock(&clone_lock); + g_free(ts); + pthread_setcancelstate(old_cancel_state, NULL); + return -1; + } + rcu_register_thread(); + new_env = cpu_copy(parent_env); + if (!new_env) { + rcu_unregister_thread(); + pthread_mutex_unlock(&clone_lock); + g_free(ts); + pthread_setcancelstate(old_cancel_state, NULL); + return -1; + } + new_cpu = env_cpu(new_env); + new_cpu->opaque = ts; +#ifdef CONFIG_LATX_FAST_JMPCACHE + new_env->tb_jmp_cache_ptr = NULL; +#endif +#ifdef CONFIG_LATX_KZT + new_env->kzt_guest_tls_parent_snapshot = NULL; + new_env->kzt_guest_tls_allocation = NULL; + new_env->kzt_guest_thread_state = NULL; + if (kzt_guest_tls_clone_parent_snapshot(parent_env, new_env) != 0) { + latx_host_thread_release_cpu(new_cpu); + rcu_unregister_thread(); + pthread_mutex_unlock(&clone_lock); + pthread_setcancelstate(old_cancel_state, NULL); + return -1; + } +#endif + new_env->kzt_guest_stack_base = 0; + stack = target_mmap(0, NEW_STACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, 0); + if (is_error(stack)) { + latx_host_thread_release_cpu(env_cpu(new_env)); + rcu_unregister_thread(); + pthread_mutex_unlock(&clone_lock); + pthread_setcancelstate(old_cancel_state, NULL); + return -1; + } + new_env->kzt_guest_stack_base = stack; + + cpu_clone_regs_child(new_env, stack + NEW_STACK_SIZE, 0); + ts->bprm = parent_ts->bprm; + ts->info = parent_ts->info; + ts->signal_mask = parent_ts->signal_mask; + ts->seccomp_filter = parent_ts->seccomp_filter; + ts->ipc_namespace_isolated = parent_ts->ipc_namespace_isolated; + new_cpu->random_seed = qemu_guest_random_seed_thread_part1(); + + tcg_register_thread(); + thread_cpu = new_cpu; + task_settid(ts); + qemu_guest_random_seed_thread_part2(new_cpu->random_seed); +#ifdef CONFIG_LATX_FAST_JMPCACHE + if (!latx_fast_jmp_cache_init(new_env)) { + fprintf(stderr, "[LATX-ERR] latx_fast_jmp_cache_init error!\n"); + } +#endif + latx_lsenv_init(new_env); + ret = kzt_guest_tls_initialize(parent_env, new_env); + if (ret != 0) { + pthread_mutex_unlock(&clone_lock); + latx_host_thread_destructor(new_cpu); + pthread_setcancelstate(old_cancel_state, NULL); + return ret; + } + kzt_guest_thread_initialize(new_env); + + if (pthread_setspecific(latx_host_thread_key, new_cpu) != 0) { + pthread_mutex_unlock(&clone_lock); + latx_host_thread_destructor(new_cpu); + pthread_setcancelstate(old_cancel_state, NULL); + return -1; + } + pthread_mutex_unlock(&clone_lock); + pthread_setcancelstate(old_cancel_state, NULL); + return 0; +} + +int latx_attach_current_host_thread(void) +{ + int result; + unsigned int delay_us = 1000; + + for (int attempt = 0; + attempt < LATX_HOST_THREAD_ATTACH_RETRIES; ++attempt) { + result = latx_attach_current_host_thread_once(); + if (result != KZT_GUEST_TLS_REFRESH_BUSY) { + return result; + } + if (attempt + 1 < LATX_HOST_THREAD_ATTACH_RETRIES) { + /* + * Wait without the attempt's locks or Guest state. Back off + * rather than repeatedly allocating CPUs and deferred caches. + */ + g_usleep(delay_us); + delay_us = MIN(delay_us * 2, 32000U); + } + } + return -1; +} +#endif /* CONFIG_LATX */ + static int do_sys_futex(int *uaddr, int op, int val, const struct timespec *timeout, int *uaddr2, int val3); @@ -9882,6 +10168,7 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp, return -TARGET_ERESTARTSYS; } + kzt_guest_tls_fork_prepare(); fork_start(); #if defined(TARGET_NR_timer_create) posix_timer_fork_start(); @@ -9924,6 +10211,9 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp, /* Child Process. */ cpu_clone_regs_child(env, newsp, flags); fork_end(1); + kzt_guest_tls_after_fork_child(env); + kzt_guest_loader_after_fork_child(); + kzt_guest_thread_after_fork_child(); #if defined(TARGET_NR_timer_create) posix_timer_fork_end(true); #endif @@ -9973,6 +10263,7 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp, } cpu_clone_regs_parent(env, flags); fork_end(0); + kzt_guest_tls_fork_parent(); #if defined(TARGET_NR_timer_create) posix_timer_fork_end(false); #endif @@ -10571,7 +10862,9 @@ void syscall_init(void) /* Build target_to_host_errno_table[] table from * host_to_target_errno_table[]. */ for (i = 0; i < ERRNO_TABLE_SIZE; i++) { - target_to_host_errno_table[host_to_target_errno_table[i]] = i; + if (host_to_target_errno_table[i]) { + target_to_host_errno_table[host_to_target_errno_table[i]] = i; + } } /* we patch the ioctl size if necessary. We rely on the fact that @@ -21149,6 +21442,148 @@ static bool syscall_user_dispatch(CPUArchState *env, int num, uint32_t arch) } #endif +#ifdef CONFIG_LATX +static bool kzt_is_fork_syscall(int num, abi_long flags) +{ + switch (num) { +#ifdef TARGET_NR_fork + case TARGET_NR_fork: +#endif +#ifdef TARGET_NR_vfork + case TARGET_NR_vfork: +#endif +#ifdef TARGET_NR_clone3 + case TARGET_NR_clone3: +#endif + return true; +#ifdef TARGET_NR_clone + case TARGET_NR_clone: + return !(flags & CLONE_VM) || (flags & CLONE_VFORK); +#endif + default: + return false; + } +} + +/* Only waits that do not change Guest TLS, mappings, or thread identity. */ +static bool kzt_syscall_can_pause(int num, abi_long operation) +{ + switch (num) { +#ifdef TARGET_NR_read + case TARGET_NR_read: +#endif +#ifdef TARGET_NR_readv + case TARGET_NR_readv: +#endif +#ifdef TARGET_NR_pread64 + case TARGET_NR_pread64: +#endif +#ifdef TARGET_NR_preadv + case TARGET_NR_preadv: +#endif +#ifdef TARGET_NR_write + case TARGET_NR_write: +#endif +#ifdef TARGET_NR_writev + case TARGET_NR_writev: +#endif +#ifdef TARGET_NR_pwrite64 + case TARGET_NR_pwrite64: +#endif +#ifdef TARGET_NR_pwritev + case TARGET_NR_pwritev: +#endif +#ifdef TARGET_NR_recvfrom + case TARGET_NR_recvfrom: +#endif +#ifdef TARGET_NR_recvmsg + case TARGET_NR_recvmsg: +#endif +#ifdef TARGET_NR_recvmmsg + case TARGET_NR_recvmmsg: +#endif +#ifdef TARGET_NR_sendto + case TARGET_NR_sendto: +#endif +#ifdef TARGET_NR_sendmsg + case TARGET_NR_sendmsg: +#endif +#ifdef TARGET_NR_accept + case TARGET_NR_accept: +#endif +#ifdef TARGET_NR_accept4 + case TARGET_NR_accept4: +#endif +#ifdef TARGET_NR_connect + case TARGET_NR_connect: +#endif +#ifdef TARGET_NR_poll + case TARGET_NR_poll: +#endif +#ifdef TARGET_NR_ppoll + case TARGET_NR_ppoll: +#endif +#ifdef TARGET_NR_select + case TARGET_NR_select: +#endif +#ifdef TARGET_NR__newselect + case TARGET_NR__newselect: +#endif +#ifdef TARGET_NR_pselect6 + case TARGET_NR_pselect6: +#endif +#ifdef TARGET_NR_epoll_wait + case TARGET_NR_epoll_wait: +#endif +#ifdef TARGET_NR_epoll_pwait + case TARGET_NR_epoll_pwait: +#endif +#ifdef TARGET_NR_epoll_pwait2 + case TARGET_NR_epoll_pwait2: +#endif +#ifdef TARGET_NR_ppoll_time64 + case TARGET_NR_ppoll_time64: +#endif +#ifdef TARGET_NR_pselect6_time64 + case TARGET_NR_pselect6_time64: +#endif +#ifdef TARGET_NR_clock_nanosleep_time64 + case TARGET_NR_clock_nanosleep_time64: +#endif +#ifdef TARGET_NR_nanosleep + case TARGET_NR_nanosleep: +#endif +#ifdef TARGET_NR_clock_nanosleep + case TARGET_NR_clock_nanosleep: +#endif +#ifdef TARGET_NR_wait4 + case TARGET_NR_wait4: +#endif +#ifdef TARGET_NR_waitpid + case TARGET_NR_waitpid: +#endif +#ifdef TARGET_NR_waitid + case TARGET_NR_waitid: +#endif +#ifdef TARGET_NR_sched_yield + case TARGET_NR_sched_yield: +#endif + return true; +#ifdef TARGET_NR_futex + case TARGET_NR_futex: +#endif +#ifdef TARGET_NR_futex_time64 + case TARGET_NR_futex_time64: +#endif + /* PI operations transfer lock ownership and require separate handling. */ + return (operation & FUTEX_CMD_MASK) == FUTEX_WAIT || + (operation & FUTEX_CMD_MASK) == FUTEX_WAIT_BITSET; + default: + return false; + } +} +#endif + abi_long do_syscall_with_seccomp(void *cpu_env, int num, int seccomp_num, uint32_t seccomp_arch, abi_long arg1, abi_long arg2, abi_long arg3, abi_long arg4, @@ -21195,6 +21630,11 @@ abi_long do_syscall_with_seccomp(void *cpu_env, int num, int seccomp_num, #ifdef TARGET_I386 if (!loader_tunnel && syscall_user_dispatch(env, seccomp_num, seccomp_arch)) { +#ifdef CONFIG_LATX + if (kzt_is_fork_syscall(num, arg1)) { + kzt_guest_tls_fork_abort(); + } +#endif return -TARGET_QEMU_ESIGRETURN; } #endif @@ -21250,6 +21690,7 @@ abi_long do_syscall_with_seccomp(void *cpu_env, int num, int seccomp_num, g_assert_not_reached(); #endif case GUEST_SECCOMP_KILL_THREAD: + kzt_guest_tls_fork_abort(); seccomp_kill_thread(env); case GUEST_SECCOMP_KILL_PROCESS: force_sig_abort(TARGET_SIGSYS); @@ -21263,6 +21704,13 @@ abi_long do_syscall_with_seccomp(void *cpu_env, int num, int seccomp_num, arg3, arg4, arg5, arg6); } +#ifdef CONFIG_LATX + /* A restarted syscall is still inside the same libc fork invocation. */ + if (kzt_is_fork_syscall(num, arg1) && + (ret != -TARGET_ERESTARTSYS || ts->seccomp_errno_return)) { + kzt_guest_tls_fork_abort(); + } +#endif record_syscall_return(cpu, num, ret); return ret; } @@ -21272,8 +21720,19 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1, abi_long arg5, abi_long arg6, abi_long arg7, abi_long arg8) { - return do_syscall_with_seccomp(cpu_env, num, num, - guest_seccomp_target_arch(), - arg1, arg2, arg3, arg4, - arg5, arg6, arg7, arg8); +#ifdef CONFIG_LATX + CPUX86State *env = cpu_env; + int tls_execution_paused = kzt_syscall_can_pause(num, arg2) + ? kzt_guest_tls_syscall_pause(env) : 0; + abi_long result = do_syscall_with_seccomp( + cpu_env, num, num, guest_seccomp_target_arch(), + arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); + + kzt_guest_tls_execution_resume(env, tls_execution_paused); + return result; +#else + return do_syscall_with_seccomp( + cpu_env, num, num, guest_seccomp_target_arch(), + arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); +#endif } diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 26aa311152..8b13f907be 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -1670,6 +1670,12 @@ typedef struct CPUX86State { #ifdef CONFIG_LATX ucontext_t *puc; uintptr_t insn_save[2]; + target_ulong kzt_guest_stack_base; +#ifdef CONFIG_LATX_KZT + void *kzt_guest_tls_allocation; + void *kzt_guest_thread_state; + void *kzt_guest_tls_parent_snapshot; +#endif #endif } CPUX86State; diff --git a/target/i386/latx/context/callback.c b/target/i386/latx/context/callback.c index 1052bbb74a..c81b01987d 100644 --- a/target/i386/latx/context/callback.c +++ b/target/i386/latx/context/callback.c @@ -19,6 +19,7 @@ #include "debug.h" #include "lsenv.h" #include "qemu.h" +#include "kzt-guest-tls.h" #ifdef TARGET_X86_64 typedef struct CallbackFrame { @@ -155,6 +156,128 @@ typedef enum LatxGuestCallKind { LATX_GUEST_INTERNAL_NO_REFRESH, } LatxGuestCallKind; +#define LATX_GUEST_TLS_REFRESH_RETRIES 1000 + +static bool callback_is_user(LatxGuestCallKind kind) +{ + return kind == LATX_GUEST_USER_CALLBACK; +} + +static int callback_disable_cancellation(LatxGuestCallKind kind, + int *old_state) +{ + if (!latx_kzt_guest_tls_enabled() || + !callback_is_user(kind)) { + return 0; + } + return pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state) == 0 + ? 1 : -1; +} + +static void callback_restore_cancellation(int disabled, int old_state, + int saved_errno, + int saved_h_errno) +{ + if (disabled > 0) { + (void)pthread_setcancelstate(old_state, NULL); + } + errno = saved_errno; + h_errno = saved_h_errno; +} + +static __thread unsigned int callback_internal_depth; + +int latx_guest_internal_callback_active(void) +{ + return callback_internal_depth != 0; +} + +typedef struct CallbackScope { + CPUX86State *cpu; + int old_cancel_state; + int cancellation_disabled; + bool execution_entered; + bool internal_entered; +} CallbackScope; + +static void callback_scope_leave(CallbackScope *scope) +{ + CallbackScope current = *scope; + int saved_errno; + int saved_h_errno; + + if (current.internal_entered) { + g_assert(callback_internal_depth > 0); + --callback_internal_depth; + } + /* Restoring cancellation may not return; cleanup must be idempotent. */ + *scope = (CallbackScope) { 0 }; + + if (current.execution_entered) { + kzt_guest_tls_execution_leave(current.cpu); + } + if (current.cancellation_disabled > 0) { + saved_errno = errno; + saved_h_errno = h_errno; + callback_restore_cancellation(current.cancellation_disabled, + current.old_cancel_state, + saved_errno, saved_h_errno); + } +} + +static int callback_scope_enter(CallbackScope *scope, LatxGuestCallKind kind) +{ + int entry_errno; + int entry_h_errno; + int result = 0; + + memset(scope, 0, sizeof(*scope)); + if (!latx_kzt_guest_tls_enabled()) { + return 0; + } + entry_errno = errno; + entry_h_errno = h_errno; + scope->cancellation_disabled = callback_disable_cancellation( + kind, &scope->old_cancel_state); + if (scope->cancellation_disabled < 0) { + return -1; + } + if ((!lsenv || !lsenv->cpu_state) && + latx_attach_current_host_thread() != 0) { + fprintf(stderr, "KZT cannot attach native thread for Guest callback\n"); + result = -1; + goto out; + } + scope->cpu = (CPUX86State *)lsenv->cpu_state; + if (!callback_is_user(kind)) { + ++callback_internal_depth; + scope->internal_entered = true; + } + if (kind != LATX_GUEST_INTERNAL_NO_REFRESH) { + for (int attempt = 0; attempt < LATX_GUEST_TLS_REFRESH_RETRIES; + ++attempt) { + result = kzt_guest_tls_refresh_if_needed(scope->cpu); + if (result != KZT_GUEST_TLS_REFRESH_BUSY || + !callback_is_user(kind)) { + break; + } + g_usleep(1000); + } + if (result != 0) { + goto out; + } + } + if (callback_is_user(kind)) { + kzt_guest_tls_execution_enter(scope->cpu); + scope->execution_entered = true; + + } +out: + errno = entry_errno; + h_errno = entry_h_errno; + return result; +} + static uint64_t run_function_with_state_va(uintptr_t fnc, int nargs, LatxGuestCallKind kind, va_list *ap) @@ -163,8 +286,11 @@ static uint64_t run_function_with_state_va(uintptr_t fnc, int nargs, size_t stack_args; CallbackFrame frame; uint64_t *stack; + CallbackScope scope __attribute__((cleanup(callback_scope_leave))) = { 0 }; - + if (callback_scope_enter(&scope, kind) != 0) { + return 0; + } lsassert(fnc); lsassert(nargs >= 0); @@ -185,7 +311,6 @@ static uint64_t run_function_with_state_va(uintptr_t fnc, int nargs, } } - (void)kind; return callback_frame_run(&frame, fnc); #else (void)fnc; @@ -240,8 +365,11 @@ uint64_t RunFunctionFmt(uintptr_t fnc, const char *fmt, ...) CallbackFrame frame; LatxCallbackArgs args; va_list ap; + CallbackScope scope __attribute__((cleanup(callback_scope_leave))) = { 0 }; - + if (callback_scope_enter(&scope, LATX_GUEST_USER_CALLBACK) != 0) { + return 0; + } lsassert(fnc); lsassert(fmt); @@ -282,8 +410,11 @@ float RunFunctionFmtFloat(uintptr_t fnc, const char *fmt, ...) CallbackFrame frame; LatxCallbackArgs args; va_list ap; + CallbackScope scope __attribute__((cleanup(callback_scope_leave))) = { 0 }; - + if (callback_scope_enter(&scope, LATX_GUEST_USER_CALLBACK) != 0) { + return 0.0f; + } lsassert(fnc); lsassert(fmt); @@ -317,3 +448,80 @@ float RunFunctionFmtFloat(uintptr_t fnc, const char *fmt, ...) return 0.0f; #endif } + +static int run_guest_callback_impl(uintptr_t entry, const long *gpr_args, + int gpr_count, const long *xmm_args, + int xmm_count, const long *stack_args, + int stack_count, long *rax, long *rdx, + long *xmm0, long *xmm1, + unsigned __int128 *st0, LatxGuestCallKind kind) +{ +#ifndef TARGET_X86_64 + return -1; +#else + CallbackFrame frame; + CallbackResult result; + uint64_t *guest_stack; + CallbackScope scope __attribute__((cleanup(callback_scope_leave))) = { 0 }; + + if (!entry || gpr_count < 0 || gpr_count > LATX_CALLBACK_GPR_ARGS || + xmm_count < 0 || xmm_count > LATX_CALLBACK_XMM_ARGS || + stack_count < 0 || (gpr_count && !gpr_args) || + (xmm_count && !xmm_args) || (stack_count && !stack_args)) { + return -1; + } + if (!latx_kzt_guest_tls_enabled() || + callback_scope_enter(&scope, kind) != 0) { + return -1; + } + + frame = callback_frame_enter((size_t)stack_count, false); + guest_stack = (uint64_t *)frame.cpu->regs[R_ESP]; + for (int index = 0; index < gpr_count; ++index) { + frame.cpu->regs[callback_gpr_regs[index]] = + (uint64_t)gpr_args[index]; + } + for (int index = 0; index < xmm_count; ++index) { + frame.cpu->xmm_regs[index].ZMM_Q(0) = + (uint64_t)xmm_args[index]; + } + for (int index = 0; index < stack_count; ++index) { + guest_stack[index] = (uint64_t)stack_args[index]; + } + + callback_frame_run_result(&frame, entry, &result); + /* + * Semantic helpers may use Guest result registers. Publish the captured + * result only after those helpers finish, even if an output aliases env. + */ + callback_scope_leave(&scope); + if (rax) { + *rax = (long)result.rax; + } + if (rdx) { + *rdx = (long)result.rdx; + } + if (xmm0) { + *xmm0 = (long)result.xmm0; + } + if (xmm1) { + *xmm1 = (long)result.xmm1; + } + if (st0) { + *st0 = result.st0; + } + return 0; +#endif +} + +int latx_run_guest_callback(uintptr_t entry, const long *gpr_args, + int gpr_count, const long *xmm_args, + int xmm_count, const long *stack_args, + int stack_count, long *rax, long *rdx, + long *xmm0, long *xmm1, + unsigned __int128 *st0) +{ + return run_guest_callback_impl(entry, gpr_args, gpr_count, xmm_args, + xmm_count, stack_args, stack_count, rax, rdx, xmm0, xmm1, st0, + LATX_GUEST_USER_CALLBACK); +} diff --git a/target/i386/latx/context/elfloader.c b/target/i386/latx/context/elfloader.c index 56f885ae6c..45a0231e76 100755 --- a/target/i386/latx/context/elfloader.c +++ b/target/i386/latx/context/elfloader.c @@ -34,6 +34,7 @@ #include "qemu.h" #include "qemu/pressure-vessel.h" #include "kzt-groups.h" +#include "kzt-guest-tls.h" #include "kzt_relocation_transaction.h" static int kzt_relocation_slot_fits_page(uintptr_t slot_addr) @@ -930,6 +931,24 @@ static int relocate_elf_rela( Elf64_Sym *sym = &head->DynSym[ELF64_R_SYM(rela[i].r_info)]; int bind = ELF64_ST_BIND(sym->st_info); const char* symname = SymName(head, sym); + if (latx_kzt_guest_tls_enabled() && transaction) { + const char *elf_name = ElfName(head); + const char *base_name = elf_name ? strrchr(elf_name, '/') : NULL; + + base_name = base_name ? base_name + 1 : elf_name; + if (base_name && + (!strcmp(base_name, "libc.so.6") || + !strncmp(base_name, "libc-", 5)) && + (!strcmp(symname, "malloc") || + !strcmp(symname, "calloc") || + !strcmp(symname, "realloc") || + !strcmp(symname, "free") || + !strcmp(symname, "memalign") || + !strcmp(symname, "aligned_alloc") || + !strcmp(symname, "posix_memalign"))) { + continue; + } + } uint64_t *p = (uint64_t*)(rela[i].r_offset + head->delta); uintptr_t offs = 0; uintptr_t end = 0; @@ -1560,6 +1579,24 @@ const char* FindNearestSymbolName(elfheader_t* h, void* p, uintptr_t* start, uin return ret; } +uintptr_t FindElfSymbolAddress(elfheader_t *h, const char *name) +{ + if (!h || !name || h->fini_done) { + return 0; + } + + for (size_t i = 0; i < h->numDynSym; ++i) { + const Elf64_Sym *sym = &h->DynSym[i]; + + if (sym->st_shndx != SHN_UNDEF && sym->st_value && + ELF64_ST_TYPE(sym->st_info) == STT_FUNC && + strcmp(h->DynStr + sym->st_name, name) == 0) { + return sym->st_value + h->delta; + } + } + return 0; +} + const char* VersionnedName(const char* name, int ver, const char* vername) { if(ver==-1) diff --git a/target/i386/latx/context/kzt-guest-thread.c b/target/i386/latx/context/kzt-guest-thread.c new file mode 100644 index 0000000000..99abb3d554 --- /dev/null +++ b/target/i386/latx/context/kzt-guest-thread.c @@ -0,0 +1,501 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include +#include +#include +#include + +#include "box64context.h" +#include "callback.h" +#include "debug.h" +#include "elfloader.h" +#include "kzt-guest-tls.h" +#include "kzt-guest-thread.h" +#include "myalign.h" +#include "qemu.h" + +#define KZT_LIBC_TSD_KEYS 1024 +#define KZT_LIBC_TSD_DESTRUCTOR_ITERATIONS 4 +#define KZT_LIBC_TSD_REFRESH_RETRIES 1000 +#define KZT_LIBC_TSD_REFRESH_RETRY_US 1000 +#define KZT_LIBC_CXA_TLS_DESTRUCTORS 1024 + +typedef struct kzt_libc_tsd_key { + uintptr_t destructor; + int in_use; +} kzt_libc_tsd_key_t; + +typedef struct kzt_libc_guest_tsd_helpers { + uintptr_t key_create; + uintptr_t key_delete; + uintptr_t setspecific; + uintptr_t getspecific; + uintptr_t cxa_thread_atexit_impl; +} kzt_libc_guest_tsd_helpers_t; + +typedef struct kzt_libc_cxa_tls_destructor { + uintptr_t destructor; + uintptr_t object; + uintptr_t dso_handle; + uintptr_t loader_hold; +} kzt_libc_cxa_tls_destructor_t; + + +typedef struct kzt_guest_thread_state { + kzt_libc_cxa_tls_destructor_t + cxa_tls_destructors[KZT_LIBC_CXA_TLS_DESTRUCTORS]; + size_t cxa_tls_destructor_count; +} kzt_guest_thread_state_t; + +static kzt_libc_guest_tsd_helpers_t *guest_tsd_helpers; +static GMutex kzt_libc_tsd_resolve_lock; +static kzt_libc_tsd_key_t kzt_libc_tsd_keys[KZT_LIBC_TSD_KEYS]; +static GMutex kzt_libc_tsd_lock; +static GMutex kzt_libc_tsd_operation_lock; + +static kzt_guest_thread_state_t *kzt_guest_thread_state(CPUX86State *env) +{ + return env ? env->kzt_guest_thread_state : NULL; +} + +void kzt_guest_thread_initialize(CPUX86State *env) +{ + g_assert(latx_kzt_guest_tls_enabled()); + g_assert(env && env->kzt_guest_tls_allocation); + env->kzt_guest_thread_state = g_new0(kzt_guest_thread_state_t, 1); +} + +void kzt_guest_thread_after_fork_child(void) +{ + GMutex *locks[] = { + &kzt_libc_tsd_resolve_lock, + &kzt_libc_tsd_operation_lock, + &kzt_libc_tsd_lock, + }; + + if (!latx_kzt_guest_tls_enabled()) { + return; + } + for (size_t i = 0; i < G_N_ELEMENTS(locks); ++i) { + memset(locks[i], 0, sizeof(*locks[i])); + g_mutex_init(locks[i]); + } +} + +static int kzt_libc_pthread_error(int error_number) +{ + return host_to_target_errno(error_number); +} + +static void QEMU_NORETURN kzt_guest_thread_abort(const char *reason) +{ + fprintf(stderr, "KZT Guest thread cleanup failed: %s\n", reason); + _exit(EXIT_FAILURE); +} + +static uintptr_t kzt_find_guest_libc_symbol(const char *name) +{ + if (!my_context || !name) { + return 0; + } + + for (int i = 0; i < my_context->elfsize; ++i) { + elfheader_t *head = my_context->elfs[i]; + const char *elf_name; + const char *base_name; + + if (!head) { + continue; + } + elf_name = ElfName(head); + if (!elf_name) { + continue; + } + base_name = strrchr(elf_name, '/'); + base_name = base_name ? base_name + 1 : elf_name; + if (strcmp(base_name, "libc.so.6") != 0 && + strncmp(base_name, "libc-", 5) != 0) { + continue; + } + return FindElfSymbolAddress(head, name); + } + return 0; +} + +static uintptr_t kzt_find_guest_pthread_symbol(const char *name) +{ + if (!my_context || !name) { + return 0; + } + + for (int pass = 0; pass < 2; ++pass) { + for (int i = 0; i < my_context->elfsize; ++i) { + elfheader_t *head = my_context->elfs[i]; + const char *elf_name; + const char *base_name; + int matches; + + if (!head) { + continue; + } + elf_name = ElfName(head); + if (!elf_name) { + continue; + } + base_name = strrchr(elf_name, '/'); + base_name = base_name ? base_name + 1 : elf_name; + if (pass == 0) { + matches = strcmp(base_name, "libc.so.6") == 0 || + strncmp(base_name, "libc-", 5) == 0; + } else { + matches = strcmp(base_name, "libpthread.so.0") == 0 || + strncmp(base_name, "libpthread-", 11) == 0; + } + if (matches) { + uintptr_t address = FindElfSymbolAddress(head, name); + + if (address) { + return address; + } + } + } + } + return 0; +} + +static int kzt_libc_resolve_guest_tsd( + const kzt_libc_guest_tsd_helpers_t **resolved) +{ + kzt_libc_guest_tsd_helpers_t candidate = { 0 }; + kzt_libc_guest_tsd_helpers_t *published; + + if (!resolved) { + return -1; + } + published = g_atomic_pointer_get(&guest_tsd_helpers); + if (published) { + *resolved = published; + return 0; + } + + g_mutex_lock(&kzt_libc_tsd_resolve_lock); + published = g_atomic_pointer_get(&guest_tsd_helpers); + if (published) { + g_mutex_unlock(&kzt_libc_tsd_resolve_lock); + *resolved = published; + return 0; + } + + candidate.key_create = + kzt_find_guest_pthread_symbol("pthread_key_create"); + candidate.key_delete = + kzt_find_guest_pthread_symbol("pthread_key_delete"); + candidate.setspecific = + kzt_find_guest_pthread_symbol("pthread_setspecific"); + candidate.getspecific = + kzt_find_guest_pthread_symbol("pthread_getspecific"); + if (!candidate.key_create) { + candidate.key_create = kzt_resolve_guest_object_symbol( + "libc.so.6", "pthread_key_create"); + } + if (!candidate.key_delete) { + candidate.key_delete = kzt_resolve_guest_object_symbol( + "libc.so.6", "pthread_key_delete"); + } + if (!candidate.setspecific) { + candidate.setspecific = kzt_resolve_guest_object_symbol( + "libc.so.6", "pthread_setspecific"); + } + if (!candidate.getspecific) { + candidate.getspecific = kzt_resolve_guest_object_symbol( + "libc.so.6", "pthread_getspecific"); + } + if (!candidate.key_create) { + candidate.key_create = kzt_resolve_guest_object_symbol( + "libpthread.so.0", "pthread_key_create"); + } + if (!candidate.key_delete) { + candidate.key_delete = kzt_resolve_guest_object_symbol( + "libpthread.so.0", "pthread_key_delete"); + } + if (!candidate.setspecific) { + candidate.setspecific = kzt_resolve_guest_object_symbol( + "libpthread.so.0", "pthread_setspecific"); + } + if (!candidate.getspecific) { + candidate.getspecific = kzt_resolve_guest_object_symbol( + "libpthread.so.0", "pthread_getspecific"); + } + candidate.cxa_thread_atexit_impl = + kzt_find_guest_libc_symbol("__cxa_thread_atexit_impl"); + if (!candidate.cxa_thread_atexit_impl) { + candidate.cxa_thread_atexit_impl = + kzt_resolve_guest_object_symbol( + "libc.so.6", "__cxa_thread_atexit_impl"); + } + if (!candidate.key_create || !candidate.key_delete || + !candidate.setspecific || !candidate.getspecific || + !candidate.cxa_thread_atexit_impl) { + printf_log(LOG_INFO, + "KZT cannot resolve Guest thread-state helpers\n"); + g_mutex_unlock(&kzt_libc_tsd_resolve_lock); + return -1; + } + published = g_new(kzt_libc_guest_tsd_helpers_t, 1); + *published = candidate; + g_atomic_pointer_set(&guest_tsd_helpers, published); + g_mutex_unlock(&kzt_libc_tsd_resolve_lock); + *resolved = published; + return 0; +} + + +int kzt_guest_thread_key_create(CPUX86State *env, + unsigned int *key, + uintptr_t destructor) +{ + const kzt_libc_guest_tsd_helpers_t *helpers; + int result; + + if (!env || !key) { + return kzt_libc_pthread_error(EINVAL); + } + if (kzt_libc_resolve_guest_tsd(&helpers) != 0) { + return kzt_libc_pthread_error(EAGAIN); + } + g_mutex_lock(&kzt_libc_tsd_operation_lock); + result = (int)RunFunctionWithStateInternalNoRefresh( + helpers->key_create, 2, + (uint64_t)(uintptr_t)key, (uint64_t)destructor); + if (result != 0) { + g_mutex_unlock(&kzt_libc_tsd_operation_lock); + return result; + } + if (*key >= KZT_LIBC_TSD_KEYS) { + (void)RunFunctionWithStateInternalNoRefresh( + helpers->key_delete, 1, (uint64_t)*key); + g_mutex_unlock(&kzt_libc_tsd_operation_lock); + return kzt_libc_pthread_error(EAGAIN); + } + + g_mutex_lock(&kzt_libc_tsd_lock); + kzt_libc_tsd_keys[*key] = (kzt_libc_tsd_key_t) { + .destructor = destructor, + .in_use = 1, + }; + g_mutex_unlock(&kzt_libc_tsd_lock); + g_mutex_unlock(&kzt_libc_tsd_operation_lock); + return 0; +} + +int kzt_guest_thread_key_delete(CPUX86State *env, + unsigned int key) +{ + const kzt_libc_guest_tsd_helpers_t *helpers; + int result; + + if (!env || kzt_libc_resolve_guest_tsd(&helpers) != 0) { + return kzt_libc_pthread_error(EINVAL); + } + g_mutex_lock(&kzt_libc_tsd_operation_lock); + result = (int)RunFunctionWithStateInternalNoRefresh( + helpers->key_delete, 1, (uint64_t)key); + if (result != 0) { + g_mutex_unlock(&kzt_libc_tsd_operation_lock); + return result; + } + if (key >= KZT_LIBC_TSD_KEYS) { + g_mutex_unlock(&kzt_libc_tsd_operation_lock); + return 0; + } + + g_mutex_lock(&kzt_libc_tsd_lock); + memset(&kzt_libc_tsd_keys[key], 0, + sizeof(kzt_libc_tsd_keys[key])); + g_mutex_unlock(&kzt_libc_tsd_lock); + g_mutex_unlock(&kzt_libc_tsd_operation_lock); + return 0; +} + +static int kzt_libc_hold_cxa_dso(uintptr_t owner_address, + uintptr_t *loader_hold) +{ + kzt_x86_64_link_map_prefix_t map; + uintptr_t link_map_addr; + void *locked_map; + char *first_name_byte; + + if (!owner_address || !loader_hold || !my_context || + !my_context->dlprivate) { + return -1; + } + *loader_hold = 0; + link_map_addr = kzt_find_guest_link_map_by_address(owner_address); + if (!link_map_addr) { + return -1; + } + locked_map = lock_user( + VERIFY_READ, (abi_ulong)link_map_addr, sizeof(map), 1); + if (!locked_map) { + return -1; + } + memcpy(&map, locked_map, sizeof(map)); + unlock_user(locked_map, (abi_ulong)link_map_addr, 0); + if (!map.name) { + return 0; + } + first_name_byte = lock_user( + VERIFY_READ, (abi_ulong)map.name, 1, 1); + if (!first_name_byte) { + return -1; + } + if (!*first_name_byte) { + unlock_user(first_name_byte, (abi_ulong)map.name, 0); + return 0; + } + unlock_user(first_name_byte, (abi_ulong)map.name, 0); + if (!my_context->dlprivate->x86dlopen || + !my_context->dlprivate->x86dlclose) { + return -1; + } + *loader_hold = kzt_guest_loader_hold_open(map.name); + return *loader_hold ? 0 : -1; +} + +int kzt_guest_thread_cxa_atexit( + CPUX86State *env, uintptr_t destructor, + uintptr_t object, uintptr_t dso_handle) +{ + const kzt_libc_guest_tsd_helpers_t *helpers; + kzt_guest_thread_state_t *state = + kzt_guest_thread_state(env); + kzt_libc_cxa_tls_destructor_t *entry; + uintptr_t loader_hold = 0; + + if (!env || !destructor || !object || + kzt_libc_resolve_guest_tsd(&helpers) != 0) { + return -1; + } + if (!env->kzt_guest_tls_allocation) { + return (int)RunFunctionWithStateInternalNoRefresh( + helpers->cxa_thread_atexit_impl, 3, + destructor, object, dso_handle); + } + if (!state || + state->cxa_tls_destructor_count == + KZT_LIBC_CXA_TLS_DESTRUCTORS || + kzt_libc_hold_cxa_dso( + dso_handle ? dso_handle : destructor, + &loader_hold) != 0) { + return -1; + } + entry = &state->cxa_tls_destructors[ + state->cxa_tls_destructor_count++]; + *entry = (kzt_libc_cxa_tls_destructor_t) { + .destructor = destructor, + .object = object, + .dso_handle = dso_handle, + .loader_hold = loader_hold, + }; + return 0; +} + +static void kzt_guest_thread_run_cxa_destructors( + kzt_guest_thread_state_t *state) +{ + while (state->cxa_tls_destructor_count) { + kzt_libc_cxa_tls_destructor_t entry = + state->cxa_tls_destructors[ + --state->cxa_tls_destructor_count]; + + memset(&state->cxa_tls_destructors[ + state->cxa_tls_destructor_count], + 0, sizeof(entry)); + RunFunctionWithStateInternalNoRefresh( + entry.destructor, 1, entry.object); + if (entry.loader_hold) { + kzt_guest_loader_hold_close(entry.loader_hold); + } + } +} + +static void kzt_guest_thread_run_tsd_destructors(CPUX86State *env) +{ + const kzt_libc_guest_tsd_helpers_t *helpers; + + if (kzt_libc_resolve_guest_tsd(&helpers) != 0) { + kzt_guest_thread_abort( + "cannot resolve Guest pthread TSD destructors"); + } + for (int iteration = 0; + iteration < KZT_LIBC_TSD_DESTRUCTOR_ITERATIONS; ++iteration) { + int called = 0; + + for (unsigned int key = 0; key < KZT_LIBC_TSD_KEYS; ++key) { + uintptr_t destructor = 0; + void *value = NULL; + + g_mutex_lock(&kzt_libc_tsd_lock); + if (kzt_libc_tsd_keys[key].in_use) { + destructor = kzt_libc_tsd_keys[key].destructor; + } + g_mutex_unlock(&kzt_libc_tsd_lock); + + if (!destructor) { + continue; + } + value = (void *)(uintptr_t) + RunFunctionWithStateInternalNoRefresh( + helpers->getspecific, 1, (uint64_t)key); + if (value) { + if ((int)RunFunctionWithStateInternalNoRefresh( + helpers->setspecific, 2, + (uint64_t)key, 0) != 0) { + kzt_guest_thread_abort( + "cannot clear Guest pthread TSD value"); + } + RunFunctionWithStateInternalNoRefresh( + destructor, 1, (uint64_t)(uintptr_t)value); + called = 1; + } + } + if (!called) { + break; + } + } +} + +void kzt_guest_thread_destroy(CPUX86State *env) +{ + kzt_guest_thread_state_t *state; + int refresh_result = -1; + + if (!env) { + return; + } + state = env->kzt_guest_thread_state; + if (!state) { + return; + } + for (int attempt = 0; + attempt < KZT_LIBC_TSD_REFRESH_RETRIES; ++attempt) { + refresh_result = kzt_guest_tls_refresh(env); + if (refresh_result == 0) { + break; + } + g_usleep(KZT_LIBC_TSD_REFRESH_RETRY_US); + } + if (refresh_result != 0) { + kzt_guest_thread_abort( + "Guest TLS refresh timed out before thread destructors"); + } + kzt_guest_tls_execution_enter(env); + kzt_guest_thread_run_cxa_destructors(state); + kzt_guest_thread_run_tsd_destructors(env); + kzt_guest_tls_execution_leave(env); + g_free(state); + env->kzt_guest_thread_state = NULL; +} diff --git a/target/i386/latx/context/kzt-guest-tls.c b/target/i386/latx/context/kzt-guest-tls.c new file mode 100644 index 0000000000..699b128b6f --- /dev/null +++ b/target/i386/latx/context/kzt-guest-tls.c @@ -0,0 +1,2401 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include +#include +#include +#include + +#include "box64context.h" +#include "callback.h" +#include "debug.h" +#include "elfloader.h" +#include "kzt-guest-tls.h" +#include "kzt-guest-tls-epoch.h" +#include "myalign.h" +#include "qemu.h" + +#define KZT_GUEST_MAX_DTV_ENTRIES 4096 +#define KZT_GUEST_MAX_STATIC_TLS_SIZE (16 * 1024 * 1024) +#define KZT_GUEST_MAX_TCB_SIZE (64 * 1024) +#define KZT_GUEST_TLS_TARGET_WAIT_RETRIES 1000 +#define KZT_GUEST_ROBUST_LIST_LIMIT 2048 + +typedef struct kzt_guest_dtv_entry { + uintptr_t value; + uintptr_t to_free; +} kzt_guest_dtv_entry_t; + +typedef struct kzt_guest_tls_state { + GRecMutex execution_lock; + GThread *execution_owner; + gint execution_depth; + gint propagation_refs; + gint destroying; + void *static_allocation; + uintptr_t static_start; + size_t static_size; + uintptr_t thread_pointer; + void *dtv_allocation; + kzt_guest_dtv_entry_t *dtv; + void *dynamic_allocations[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + unsigned char + dynamic_guest_owned[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + size_t dynamic_module_ids[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + kzt_public_loader_tls_object_t + dynamic_objects[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + size_t dynamic_module_count; + kzt_public_loader_tls_object_t + tls_inventory[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + size_t tls_inventory_count; + uintptr_t inventory_generation; + kzt_public_loader_tls_object_t + preinitialized_static_objects[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + size_t preinitialized_static_count; + uintptr_t robust_head_addr; + uintptr_t robust_prev_addr; + intptr_t robust_futex_offset; + int refreshing; + struct kzt_guest_tls_state *next; +} kzt_guest_tls_state_t; + +typedef struct kzt_x86_64_tcbhead { + uintptr_t tcb; + kzt_guest_dtv_entry_t *dtv; + uintptr_t self; + int multiple_threads; + int gscope_flag; + uintptr_t sysinfo; + uintptr_t stack_guard; + uintptr_t pointer_guard; + uintptr_t vgetcpu_cache[2]; + unsigned int feature_1; + int unused_1; +} kzt_x86_64_tcbhead_t; + +typedef struct kzt_guest_parent_tls_snapshot { + uintptr_t parent_tp; + size_t tcb_bytes_size; + size_t dtv_capacity; + size_t robust_head_offset; + size_t robust_prev_offset; + size_t tid_offset; + intptr_t robust_futex_offset; + int robust_head_valid; + unsigned char tcb_bytes[KZT_GUEST_MAX_TCB_SIZE + sizeof(uintptr_t)]; +} kzt_guest_parent_tls_snapshot_t; + +static uintptr_t guest_uselocale; +static uintptr_t guest_ctype_init; +static uintptr_t guest_get_tls_static_info; +static uintptr_t guest_allocate_tls_init; +static uintptr_t guest_free; +static uintptr_t guest_dlinfo; +static GMutex kzt_guest_tls_states_lock; +static GRecMutex kzt_guest_tls_refresh_lock; +static gsize kzt_guest_tls_refresh_lock_initialized; +static GRWLock kzt_guest_tls_fork_lock; +static gsize kzt_guest_tls_fork_lock_initialized; +static __thread unsigned int kzt_guest_tls_global_execution_depth; +static __thread int kzt_guest_tls_fork_reader_paused; +static __thread int kzt_guest_tls_fork_writer_owned; +static kzt_guest_tls_state_t *kzt_guest_tls_states; +static kzt_public_loader_tls_object_t + kzt_guest_tls_process_inventory[KZT_PUBLIC_LOADER_MAX_OBJECTS]; +static size_t kzt_guest_tls_process_inventory_count; +static kzt_public_loader_tls_object_t + kzt_guest_tls_loader_inventory[KZT_PUBLIC_LOADER_MAX_OBJECTS]; +static size_t kzt_guest_tls_loader_inventory_count; +static uintptr_t kzt_guest_tls_loader_generation; +static int kzt_guest_tls_loader_inventory_initialized; +static uintptr_t kzt_guest_tls_inventory_generation; +static uintptr_t kzt_guest_tls_pending_propagation_generation; +static int kzt_guest_tls_process_inventory_initialized; +/* Zero disables reuse, odd marks a loader transition, even is stable. */ +static gint kzt_guest_tls_loader_sequence; + +void kzt_guest_tls_loader_tracking_enable(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + (void)g_atomic_int_compare_and_exchange( + &kzt_guest_tls_loader_sequence, 0, 1); +} + +void kzt_guest_tls_loader_tracking_reset(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + g_atomic_int_set(&kzt_guest_tls_loader_sequence, 0); + memset(kzt_guest_tls_loader_inventory, 0, + sizeof(kzt_guest_tls_loader_inventory)); + kzt_guest_tls_loader_inventory_count = 0; + qatomic_set(&kzt_guest_tls_loader_generation, 0); + kzt_guest_tls_loader_inventory_initialized = 0; +} + +void kzt_guest_tls_loader_event_begin(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + gint sequence; + uint32_t dirty; + + do { + sequence = g_atomic_int_get(&kzt_guest_tls_loader_sequence); + dirty = kzt_guest_tls_epoch_begin_change((uint32_t)sequence); + if (!dirty || dirty == (uint32_t)sequence) { + return; + } + } while (!g_atomic_int_compare_and_exchange( + &kzt_guest_tls_loader_sequence, + sequence, (gint)dirty)); +} + +static void kzt_guest_tls_publish_stable_epoch(void) +{ + gint sequence; + uint32_t stable; + + if (qatomic_read(&kzt_guest_tls_pending_propagation_generation) || + !kzt_guest_loader_state_is_consistent()) { + return; + } + do { + sequence = g_atomic_int_get(&kzt_guest_tls_loader_sequence); + stable = kzt_guest_tls_epoch_publish_stable((uint32_t)sequence); + if (!stable || stable == (uint32_t)sequence) { + return; + } + } while (!g_atomic_int_compare_and_exchange( + &kzt_guest_tls_loader_sequence, + sequence, (gint)stable)); +} + +static int kzt_guest_tls_can_reuse_snapshot( + const kzt_guest_tls_state_t *state) +{ + uint32_t sequence_before = (uint32_t)g_atomic_int_get( + &kzt_guest_tls_loader_sequence); + uintptr_t pending_generation = qatomic_read( + &kzt_guest_tls_pending_propagation_generation); + uintptr_t process_generation = qatomic_read( + &kzt_guest_tls_inventory_generation); + int loader_consistent = kzt_guest_loader_state_is_consistent(); + uint32_t sequence_after = (uint32_t)g_atomic_int_get( + &kzt_guest_tls_loader_sequence); + + return state->dtv && kzt_guest_tls_epoch_can_reuse( + sequence_before, sequence_after, loader_consistent, + pending_generation, state->inventory_generation, + process_generation); +} + +static void kzt_guest_tls_initialize_fork_lock(void) +{ + if (g_once_init_enter(&kzt_guest_tls_fork_lock_initialized)) { + g_rw_lock_init(&kzt_guest_tls_fork_lock); + g_once_init_leave(&kzt_guest_tls_fork_lock_initialized, 1); + } +} + +static void kzt_guest_tls_initialize_refresh_lock(void) +{ + if (g_once_init_enter(&kzt_guest_tls_refresh_lock_initialized)) { + g_rec_mutex_init(&kzt_guest_tls_refresh_lock); + g_once_init_leave(&kzt_guest_tls_refresh_lock_initialized, 1); + } +} + +void kzt_guest_tls_execution_enter(CPUX86State *env) +{ + kzt_guest_tls_state_t *state; + GThread *current; + + if (!env || !env->kzt_guest_tls_allocation) { + return; + } + state = env->kzt_guest_tls_allocation; + current = g_thread_self(); + kzt_guest_tls_initialize_fork_lock(); + if (!kzt_guest_tls_global_execution_depth++) { + g_rw_lock_reader_lock(&kzt_guest_tls_fork_lock); + } + g_rec_mutex_lock(&state->execution_lock); + g_assert(!g_atomic_int_get(&state->execution_depth) || + g_atomic_pointer_get(&state->execution_owner) == current); + g_atomic_pointer_set(&state->execution_owner, current); + g_atomic_int_inc(&state->execution_depth); +} + +void kzt_guest_tls_execution_leave(CPUX86State *env) +{ + kzt_guest_tls_state_t *state; + GThread *current; + + if (!env || !env->kzt_guest_tls_allocation) { + return; + } + state = env->kzt_guest_tls_allocation; + current = g_thread_self(); + g_assert(g_atomic_int_get(&state->execution_depth) > 0 && + g_atomic_pointer_get(&state->execution_owner) == current); + if (g_atomic_int_dec_and_test(&state->execution_depth)) { + g_atomic_pointer_set(&state->execution_owner, NULL); + } + g_rec_mutex_unlock(&state->execution_lock); + g_assert(kzt_guest_tls_global_execution_depth > 0); + if (!--kzt_guest_tls_global_execution_depth) { + g_rw_lock_reader_unlock(&kzt_guest_tls_fork_lock); + } +} + +int kzt_guest_tls_execution_pause(CPUX86State *env) +{ + kzt_guest_tls_state_t *state; + int execution_depth; + + if (!env || !env->kzt_guest_tls_allocation) { + return 0; + } + kzt_guest_tls_loader_event_begin(); + state = env->kzt_guest_tls_allocation; + execution_depth = g_atomic_int_get(&state->execution_depth); + if (!execution_depth || + g_atomic_pointer_get(&state->execution_owner) != + g_thread_self()) { + return 0; + } + g_atomic_pointer_set(&state->execution_owner, NULL); + g_atomic_int_set(&state->execution_depth, 0); + for (int index = 0; index < execution_depth; ++index) { + g_rec_mutex_unlock(&state->execution_lock); + } + g_assert(kzt_guest_tls_global_execution_depth == + (unsigned int)execution_depth); + kzt_guest_tls_global_execution_depth = 0; + g_rw_lock_reader_unlock(&kzt_guest_tls_fork_lock); + return execution_depth; +} + +void kzt_guest_tls_execution_resume(CPUX86State *env, int paused) +{ + for (int index = 0; index < paused; ++index) { + kzt_guest_tls_execution_enter(env); + } +} + +static void kzt_guest_tls_fork_lock_writer(void) +{ + kzt_guest_tls_initialize_fork_lock(); + g_assert(!kzt_guest_tls_fork_reader_paused); + if (kzt_guest_tls_global_execution_depth) { + g_rw_lock_reader_unlock(&kzt_guest_tls_fork_lock); + kzt_guest_tls_fork_reader_paused = 1; + } + /* + * Do not queue a blocking writer. A loader waiter may have paused its + * read side and must be allowed to reacquire it before that outer Guest + * execution can drain. Polling leaves that recovery path runnable. + */ + while (!g_rw_lock_writer_trylock(&kzt_guest_tls_fork_lock)) { + g_usleep(1000); + } + kzt_guest_tls_fork_writer_owned = 1; +} + +void kzt_guest_tls_fork_prepare_early(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + g_assert(!kzt_guest_tls_fork_writer_owned); + kzt_guest_tls_fork_lock_writer(); +} + +void kzt_guest_tls_fork_prepare(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + if (!kzt_guest_tls_fork_writer_owned) { + kzt_guest_tls_fork_lock_writer(); + } +} + +void kzt_guest_tls_fork_parent(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + g_assert(kzt_guest_tls_fork_writer_owned); + g_rw_lock_writer_unlock(&kzt_guest_tls_fork_lock); + kzt_guest_tls_fork_writer_owned = 0; + if (kzt_guest_tls_fork_reader_paused) { + g_rw_lock_reader_lock(&kzt_guest_tls_fork_lock); + kzt_guest_tls_fork_reader_paused = 0; + } +} + +void kzt_guest_tls_fork_abort(void) +{ + if (latx_kzt_guest_tls_enabled() && kzt_guest_tls_fork_writer_owned) { + kzt_guest_tls_fork_parent(); + } +} + +int kzt_guest_tls_syscall_pause(CPUX86State *env) +{ + kzt_guest_tls_state_t *state = env ? env->kzt_guest_tls_allocation : NULL; + + /* Internal helpers can hold loader/inventory locks across their syscalls. */ + if (!state || state->refreshing || kzt_guest_tls_fork_writer_owned || + latx_guest_internal_callback_active()) { + return 0; + } + /* CallbackScope keeps cancellation disabled and owns this CPU/TLS state. */ + return kzt_guest_tls_execution_pause(env); +} + +void kzt_guest_tls_after_fork_child(CPUX86State *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + kzt_guest_tls_state_t *current = env + ? env->kzt_guest_tls_allocation : NULL; + GThread *thread = g_thread_self(); + int execution_depth = current + ? g_atomic_int_get(¤t->execution_depth) : 0; + + memset(&kzt_guest_tls_fork_lock, 0, + sizeof(kzt_guest_tls_fork_lock)); + g_rw_lock_init(&kzt_guest_tls_fork_lock); + kzt_guest_tls_fork_lock_initialized = 1; + kzt_guest_tls_fork_reader_paused = 0; + kzt_guest_tls_fork_writer_owned = 0; + g_atomic_int_set( + &kzt_guest_tls_loader_sequence, + (gint)kzt_guest_tls_epoch_after_fork((uint32_t)g_atomic_int_get( + &kzt_guest_tls_loader_sequence))); + if (kzt_guest_tls_global_execution_depth) { + g_rw_lock_reader_lock(&kzt_guest_tls_fork_lock); + } + memset(&kzt_guest_tls_states_lock, 0, + sizeof(kzt_guest_tls_states_lock)); + g_mutex_init(&kzt_guest_tls_states_lock); + memset(&kzt_guest_tls_refresh_lock, 0, + sizeof(kzt_guest_tls_refresh_lock)); + g_rec_mutex_init(&kzt_guest_tls_refresh_lock); + kzt_guest_tls_refresh_lock_initialized = 1; + if (current) { + memset(¤t->execution_lock, 0, + sizeof(current->execution_lock)); + g_rec_mutex_init(¤t->execution_lock); + g_atomic_pointer_set(¤t->execution_owner, + execution_depth ? thread : NULL); + g_atomic_int_set(¤t->execution_depth, 0); + for (int index = 0; index < execution_depth; ++index) { + g_rec_mutex_lock(¤t->execution_lock); + g_atomic_int_inc(¤t->execution_depth); + } + current->next = NULL; + g_atomic_int_set(¤t->propagation_refs, 0); + g_atomic_int_set(¤t->destroying, 0); + kzt_guest_tls_states = current; + memcpy(kzt_guest_tls_process_inventory, + current->tls_inventory, + current->tls_inventory_count * + sizeof(current->tls_inventory[0])); + kzt_guest_tls_process_inventory_count = + current->tls_inventory_count; + qatomic_set(&kzt_guest_tls_inventory_generation, + current->inventory_generation); + memcpy(kzt_guest_tls_loader_inventory, + current->tls_inventory, + current->tls_inventory_count * + sizeof(current->tls_inventory[0])); + kzt_guest_tls_loader_inventory_count = + current->tls_inventory_count; + qatomic_set(&kzt_guest_tls_loader_generation, + current->dtv ? current->dtv[0].value : 0); + kzt_guest_tls_loader_inventory_initialized = 1; + qatomic_set(&kzt_guest_tls_pending_propagation_generation, 0); + kzt_guest_tls_process_inventory_initialized = 1; + } else { + kzt_guest_tls_states = NULL; + memset(kzt_guest_tls_process_inventory, 0, + sizeof(kzt_guest_tls_process_inventory)); + kzt_guest_tls_process_inventory_count = 0; + memset(kzt_guest_tls_loader_inventory, 0, + sizeof(kzt_guest_tls_loader_inventory)); + kzt_guest_tls_loader_inventory_count = 0; + qatomic_set(&kzt_guest_tls_loader_generation, 0); + kzt_guest_tls_loader_inventory_initialized = 0; + qatomic_set(&kzt_guest_tls_inventory_generation, 0); + qatomic_set(&kzt_guest_tls_pending_propagation_generation, 0); + kzt_guest_tls_process_inventory_initialized = 0; + } +} + +static uintptr_t kzt_find_guest_libc_symbol(const char *name) +{ + if (!my_context || !name) { + return 0; + } + + for (int i = 0; i < my_context->elfsize; ++i) { + elfheader_t *head = my_context->elfs[i]; + const char *elf_name; + const char *base_name; + + if (!head) { + continue; + } + elf_name = ElfName(head); + if (!elf_name) { + continue; + } + base_name = strrchr(elf_name, '/'); + base_name = base_name ? base_name + 1 : elf_name; + if (strcmp(base_name, "libc.so.6") != 0 && + strncmp(base_name, "libc-", 5) != 0) { + continue; + } + return FindElfSymbolAddress(head, name); + } + return 0; +} + +static int kzt_guest_tls_resolve_loader(void) +{ + if (!guest_get_tls_static_info) { + guest_get_tls_static_info = + kzt_resolve_guest_symbol("_dl_get_tls_static_info"); + if (!guest_get_tls_static_info) { + guest_get_tls_static_info = + kzt_resolve_guest_object_symbol( + "ld-linux-x86-64.so.2", + "_dl_get_tls_static_info"); + } + } + if (!guest_allocate_tls_init) { + guest_allocate_tls_init = + kzt_resolve_guest_symbol("_dl_allocate_tls_init"); + if (!guest_allocate_tls_init) { + guest_allocate_tls_init = + kzt_resolve_guest_object_symbol( + "ld-linux-x86-64.so.2", + "_dl_allocate_tls_init"); + } + } + if (!guest_free) { + guest_free = kzt_resolve_guest_symbol("free"); + if (!guest_free) { + guest_free = kzt_find_guest_libc_symbol("free"); + } + } + if (!guest_get_tls_static_info || !guest_allocate_tls_init || + !guest_free) { + printf_log(LOG_INFO, + "KZT Guest TLS cannot resolve loader helpers: " + "static_info=%p allocate_init=%p free=%p\n", + (void *)guest_get_tls_static_info, + (void *)guest_allocate_tls_init, + (void *)guest_free); + return -1; + } + return 0; +} + +int kzt_guest_tls_snapshot_parent(CPUX86State *parent, + CPUX86State *child) +{ + if (!latx_kzt_guest_tls_enabled()) { + return -1; + } + kzt_guest_parent_tls_snapshot_t *snapshot; + kzt_x86_64_tcbhead_t *tcb; + kzt_guest_dtv_entry_t *capacity_entry; + uintptr_t parent_tp; + struct robust_list_head *registered_head = NULL; + size_t registered_length = 0; + size_t offset; + + if (!parent || !child || !parent->segs[R_FS].base) { + return -1; + } + if (kzt_guest_tls_resolve_loader() != 0) { + return -1; + } + parent_tp = parent->segs[R_FS].base; + snapshot = g_new0(kzt_guest_parent_tls_snapshot_t, 1); + snapshot->parent_tp = parent_tp; + for (offset = 0; offset < sizeof(snapshot->tcb_bytes); + offset += sizeof(uintptr_t)) { + uintptr_t *word = lock_user( + VERIFY_READ, (abi_ulong)(parent_tp + offset), + sizeof(*word), 1); + + if (!word) { + break; + } + memcpy(snapshot->tcb_bytes + offset, word, sizeof(*word)); + unlock_user(word, (abi_ulong)(parent_tp + offset), 0); + snapshot->tcb_bytes_size = offset + sizeof(*word); + } + if (snapshot->tcb_bytes_size < sizeof(kzt_x86_64_tcbhead_t)) { + g_free(snapshot); + return -1; + } + tcb = (kzt_x86_64_tcbhead_t *)snapshot->tcb_bytes; + if (!tcb->dtv) { + g_free(snapshot); + return -1; + } + capacity_entry = lock_user( + VERIFY_READ, + (abi_ulong)((uintptr_t)tcb->dtv - sizeof(*capacity_entry)), + sizeof(*capacity_entry), 1); + if (!capacity_entry) { + g_free(snapshot); + return -1; + } + snapshot->dtv_capacity = capacity_entry->value; + unlock_user( + capacity_entry, + (abi_ulong)((uintptr_t)tcb->dtv - sizeof(*capacity_entry)), 0); + if (snapshot->dtv_capacity > KZT_GUEST_MAX_DTV_ENTRIES) { + g_free(snapshot); + return -1; + } + if (syscall(SYS_get_robust_list, 0, ®istered_head, + ®istered_length) == 0 && + registered_head && + registered_length == sizeof(*registered_head) && + (uintptr_t)registered_head >= parent_tp) { + size_t head_offset = + (uintptr_t)registered_head - parent_tp; + + if (head_offset >= sizeof(uintptr_t) && + head_offset <= snapshot->tcb_bytes_size - + sizeof(*registered_head)) { + struct robust_list_head *head = + (struct robust_list_head *)(snapshot->tcb_bytes + + head_offset); + uintptr_t robust_prev; + uint32_t current_tid = (uint32_t)syscall(SYS_gettid); + size_t tid_offset = 0; + size_t tid_matches = 0; + + memcpy(&robust_prev, + snapshot->tcb_bytes + head_offset - + sizeof(robust_prev), + sizeof(robust_prev)); + for (size_t candidate = + head_offset > 64 ? head_offset - 64 : 0; + candidate + sizeof(current_tid) <= head_offset; + candidate += sizeof(current_tid)) { + uint32_t candidate_tid; + + memcpy(&candidate_tid, + snapshot->tcb_bytes + candidate, + sizeof(candidate_tid)); + if (candidate_tid == current_tid) { + tid_offset = candidate; + ++tid_matches; + } + } + if ((uintptr_t)head->list.next == + (uintptr_t)registered_head && + !head->list_op_pending && + robust_prev == (uintptr_t)registered_head && + current_tid && tid_matches == 1 && + head->futex_offset != INTPTR_MIN) { + snapshot->robust_head_offset = head_offset; + snapshot->robust_prev_offset = + head_offset - sizeof(robust_prev); + snapshot->robust_futex_offset = + head->futex_offset; + snapshot->tid_offset = tid_offset; + snapshot->robust_head_valid = 1; + } + } + } + if (!snapshot->robust_head_valid) { + g_free(snapshot); + return -1; + } + child->kzt_guest_tls_parent_snapshot = snapshot; + return 0; +} + +int kzt_guest_tls_clone_parent_snapshot(const CPUX86State *parent, + CPUX86State *child) +{ + if (!latx_kzt_guest_tls_enabled()) { + return -1; + } + const kzt_guest_parent_tls_snapshot_t *parent_snapshot; + + if (!parent || !child || child->kzt_guest_tls_parent_snapshot) { + return -1; + } + parent_snapshot = parent->kzt_guest_tls_parent_snapshot; + if (!parent_snapshot) { + return -1; + } + child->kzt_guest_tls_parent_snapshot = + g_new(kzt_guest_parent_tls_snapshot_t, 1); + memcpy(child->kzt_guest_tls_parent_snapshot, parent_snapshot, + sizeof(*parent_snapshot)); + return 0; +} + +static void kzt_guest_tls_release_parent_snapshot(CPUX86State *env) +{ + if (env) { + g_free(env->kzt_guest_tls_parent_snapshot); + env->kzt_guest_tls_parent_snapshot = NULL; + } +} + +static size_t kzt_guest_tls_find_tcb_size( + const kzt_guest_parent_tls_snapshot_t *snapshot, + size_t static_size, + size_t static_align) +{ + size_t limit = static_size < KZT_GUEST_MAX_TCB_SIZE + ? static_size : KZT_GUEST_MAX_TCB_SIZE; + size_t match = 0; + + for (size_t candidate = sizeof(kzt_x86_64_tcbhead_t); + candidate <= limit; candidate += sizeof(uintptr_t)) { + uintptr_t allocation; + uintptr_t aligned; + + if (candidate + sizeof(allocation) > + snapshot->tcb_bytes_size) { + break; + } + memcpy(&allocation, snapshot->tcb_bytes + candidate, + sizeof(allocation)); + if (!allocation || + allocation > UINTPTR_MAX - (static_align - 1)) { + continue; + } + aligned = (allocation + static_align - 1) & + ~(uintptr_t)(static_align - 1); + if (aligned <= snapshot->parent_tp && static_size >= candidate && + snapshot->parent_tp - aligned == static_size - candidate) { + if (match) { + return 0; + } + match = candidate; + } + } + return match; +} + +static void kzt_guest_tls_release_loader_state( + kzt_guest_tls_state_t *state) +{ + if (!state) { + return; + } + g_free(state->dtv_allocation); + state->dtv_allocation = NULL; + state->dtv = NULL; +} + +static int kzt_guest_tls_same_object_ignoring_module( + const kzt_public_loader_tls_object_t *left, + const kzt_public_loader_tls_object_t *right) +{ + return left->link_map_addr == right->link_map_addr && + left->load_bias == right->load_bias && + left->dynamic_addr == right->dynamic_addr && + left->image_addr == right->image_addr && + left->file_size == right->file_size && + left->memory_size == right->memory_size && + left->alignment == right->alignment && + left->first_byte_offset == right->first_byte_offset && + left->static_tls_offset == right->static_tls_offset && + left->static_tls_symbol_value == + right->static_tls_symbol_value && + left->static_tls_symbol_name_addr == + right->static_tls_symbol_name_addr && + left->load_generation == right->load_generation && + left->external_registration == right->external_registration && + left->static_tls_offset_valid == + right->static_tls_offset_valid && + left->static_tls_offset_needs_validation == + right->static_tls_offset_needs_validation && + left->static_tls_offset_pending == + right->static_tls_offset_pending; +} + +static int kzt_guest_tls_same_object( + const kzt_public_loader_tls_object_t *left, + const kzt_public_loader_tls_object_t *right) +{ + return kzt_guest_tls_same_object_ignoring_module(left, right) && + left->module_id == right->module_id; +} + +static int kzt_guest_tls_find_dynamic_index( + const kzt_guest_tls_state_t *state, + size_t module_id) +{ + for (size_t index = 0; index < state->dynamic_module_count; ++index) { + if (state->dynamic_module_ids[index] == module_id) { + return (int)index; + } + } + return -1; +} + +static void kzt_guest_tls_free_dynamic( + kzt_guest_tls_state_t *state, size_t index) +{ + void *allocation = state->dynamic_allocations[index]; + + if (!allocation) { + return; + } + if (state->dynamic_guest_owned[index]) { + (void)RunFunctionWithStateInternalNoRefresh( + guest_free, 1, (uint64_t)(uintptr_t)allocation); + } else { + g_free(allocation); + } + state->dynamic_allocations[index] = NULL; + state->dynamic_module_ids[index] = 0; + state->dynamic_guest_owned[index] = 0; + memset(&state->dynamic_objects[index], 0, + sizeof(state->dynamic_objects[index])); +} + +static void kzt_guest_tls_retire_dynamic( + kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *tls_objects, + size_t tls_object_count) +{ + for (size_t allocation_index = 0; + allocation_index < state->dynamic_module_count; + ++allocation_index) { + size_t module_id = state->dynamic_module_ids[allocation_index]; + int found = 0; + + if (!module_id || + !state->dynamic_allocations[allocation_index]) { + continue; + } + for (size_t object_index = 0; + object_index < tls_object_count; ++object_index) { + if (tls_objects[object_index].module_id == module_id && + kzt_guest_tls_same_object( + &tls_objects[object_index], + &state->dynamic_objects[allocation_index])) { + found = 1; + break; + } + } + if (found) { + continue; + } + kzt_guest_tls_free_dynamic(state, allocation_index); + if (module_id <= KZT_GUEST_MAX_DTV_ENTRIES) { + state->dtv[module_id].value = 0; + state->dtv[module_id].to_free = 0; + } + } +} + +static int kzt_guest_tls_inventory_matches( + const kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *objects, + size_t object_count) +{ + if (state->tls_inventory_count != object_count) { + return 0; + } + + for (size_t index = 0; index < object_count; ++index) { + const kzt_public_loader_tls_object_t *previous = + &state->tls_inventory[index]; + const kzt_public_loader_tls_object_t *current = &objects[index]; + + if (previous->image_addr != current->image_addr || + previous->link_map_addr != current->link_map_addr || + previous->load_bias != current->load_bias || + previous->dynamic_addr != current->dynamic_addr || + previous->file_size != current->file_size || + previous->memory_size != current->memory_size || + previous->alignment != current->alignment || + previous->first_byte_offset != + current->first_byte_offset || + previous->static_tls_offset != + current->static_tls_offset || + previous->static_tls_symbol_value != + current->static_tls_symbol_value || + previous->static_tls_symbol_name_addr != + current->static_tls_symbol_name_addr || + previous->module_id != current->module_id || + previous->load_generation != current->load_generation || + previous->external_registration != + current->external_registration || + previous->static_tls_offset_valid != + current->static_tls_offset_valid || + previous->static_tls_offset_needs_validation != + current->static_tls_offset_needs_validation || + previous->static_tls_offset_pending != + current->static_tls_offset_pending) { + return 0; + } + } + return 1; +} + +static int kzt_guest_tls_inventory_has_additions( + const kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *objects, + size_t object_count, + int *loader_additions) +{ + int additions = 0; + + *loader_additions = 0; + for (size_t index = 0; index < object_count; ++index) { + int found = 0; + + for (size_t previous = 0; + previous < state->tls_inventory_count; ++previous) { + if (kzt_guest_tls_same_object( + &objects[index], + &state->tls_inventory[previous])) { + found = 1; + break; + } + } + if (!found) { + additions = 1; + if (!objects[index].external_registration) { + *loader_additions = 1; + } + } + } + return additions; +} + +static int kzt_guest_tls_process_inventory_matches( + const kzt_public_loader_tls_object_t *objects, + size_t object_count) +{ + if (!kzt_guest_tls_process_inventory_initialized || + kzt_guest_tls_process_inventory_count != object_count) { + return 0; + } + for (size_t index = 0; index < object_count; ++index) { + if (!kzt_guest_tls_same_object( + &kzt_guest_tls_process_inventory[index], + &objects[index])) { + return 0; + } + } + return 1; +} + +static int kzt_guest_tls_update_process_inventory( + const kzt_public_loader_tls_object_t *objects, + size_t object_count) +{ + if (kzt_guest_tls_process_inventory_matches( + objects, object_count)) { + return 0; + } + if (!kzt_guest_tls_process_inventory_initialized || + qatomic_read(&kzt_guest_tls_inventory_generation) == UINTPTR_MAX) { + return -1; + } + qatomic_inc(&kzt_guest_tls_inventory_generation); + memcpy(kzt_guest_tls_process_inventory, objects, + object_count * sizeof(*objects)); + kzt_guest_tls_process_inventory_count = object_count; + return 1; +} + +static int kzt_guest_tls_loader_inventory_matches( + const kzt_public_loader_tls_object_t *objects, + size_t object_count) +{ + size_t previous = 0; + size_t current = 0; + + while (1) { + while (previous < kzt_guest_tls_loader_inventory_count && + kzt_guest_tls_loader_inventory[previous] + .external_registration) { + ++previous; + } + while (current < object_count && + objects[current].external_registration) { + ++current; + } + if (previous == kzt_guest_tls_loader_inventory_count || + current == object_count) { + return previous == kzt_guest_tls_loader_inventory_count && + current == object_count; + } + if (!kzt_guest_tls_same_object_ignoring_module( + &kzt_guest_tls_loader_inventory[previous], + &objects[current])) { + return 0; + } + ++previous; + ++current; + } +} + +int kzt_guest_tls_loader_event_observe(void) +{ + kzt_public_loader_tls_object_t + objects[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + size_t object_count = 0; + int result; + + if (!latx_kzt_guest_tls_enabled()) { + return 0; + } + mmap_lock(); + g_mutex_lock(&kzt_guest_tls_states_lock); + result = kzt_collect_guest_tls_objects( + objects, KZT_PUBLIC_LOADER_MAX_OBJECTS, &object_count); + if (result != 0 || !kzt_guest_tls_loader_inventory_initialized || + kzt_guest_tls_loader_inventory_matches(objects, object_count)) { + goto out; + } + if (qatomic_read(&kzt_guest_tls_loader_generation) == UINTPTR_MAX) { + result = -1; + goto out; + } + memcpy(kzt_guest_tls_loader_inventory, objects, + object_count * sizeof(*objects)); + kzt_guest_tls_loader_inventory_count = object_count; + qatomic_inc(&kzt_guest_tls_loader_generation); +out: + g_mutex_unlock(&kzt_guest_tls_states_lock); + mmap_unlock(); + return result; +} + +static int kzt_guest_tls_restore_known_module_id( + const kzt_guest_tls_state_t *state, + kzt_public_loader_tls_object_t *object) +{ + for (size_t index = 0; index < state->tls_inventory_count; ++index) { + const kzt_public_loader_tls_object_t *previous = + &state->tls_inventory[index]; + + if (previous->module_id && + previous->link_map_addr == object->link_map_addr && + previous->load_bias == object->load_bias && + previous->dynamic_addr == object->dynamic_addr && + previous->image_addr == object->image_addr && + previous->file_size == object->file_size && + previous->memory_size == object->memory_size && + previous->alignment == object->alignment && + previous->first_byte_offset == + object->first_byte_offset && + previous->static_tls_offset == + object->static_tls_offset && + previous->static_tls_symbol_value == + object->static_tls_symbol_value && + previous->static_tls_symbol_name_addr == + object->static_tls_symbol_name_addr && + previous->load_generation == object->load_generation && + previous->external_registration == + object->external_registration && + previous->static_tls_offset_valid == + object->static_tls_offset_valid && + previous->static_tls_offset_needs_validation == + object->static_tls_offset_needs_validation && + previous->static_tls_offset_pending == + object->static_tls_offset_pending) { + object->module_id = previous->module_id; + return 1; + } + } + return 0; +} + +static int kzt_guest_tls_resolve_new_module_ids( + const kzt_guest_tls_state_t *state, + kzt_public_loader_tls_object_t *objects, + size_t object_count, + int static_only) +{ + /* + * A default-visible TLS definition may be interposed, so the static + * relocation scan deliberately cannot attribute its DTPMOD64 value to + * the defining object. Once dlopen has returned, ask the Guest loader + * for the authoritative module ID of each newly observed TLS object. + */ + for (size_t index = 0; index < object_count; ++index) { + size_t module_id = 0; + uint64_t result; + + if (static_only && + !objects[index].static_tls_offset_valid) { + continue; + } + if (objects[index].module_id || + kzt_guest_tls_restore_known_module_id( + state, &objects[index])) { + continue; + } + if (!guest_dlinfo) { + guest_dlinfo = kzt_resolve_guest_symbol("dlinfo"); + } + if (!guest_dlinfo) { + guest_dlinfo = kzt_resolve_guest_symbol("__dlinfo"); + } + if (!guest_dlinfo && my_context && my_context->dlprivate) { + guest_dlinfo = (uintptr_t)my_context->dlprivate->x86dlinfo; + } + if (!guest_dlinfo) { + fprintf(stderr, + "KZT Guest TLS requires a loaded Guest dlinfo " + "provider to resolve TLS module IDs (object %p)\n", + (void *)objects[index].link_map_addr); + return -1; + } + result = RunFunctionWithStateInternalNoRefresh( + guest_dlinfo, 3, objects[index].link_map_addr, + RTLD_DI_TLS_MODID, (uint64_t)(uintptr_t)&module_id); + if (result != 0 || !module_id || + module_id > KZT_GUEST_MAX_DTV_ENTRIES) { + printf_log(LOG_INFO, + "KZT Guest TLS cannot query module ID for " + "late object %p\n", + (void *)objects[index].link_map_addr); + return -1; + } + objects[index].module_id = module_id; + } + return 0; +} + +static int kzt_guest_tls_validate_inventory( + const kzt_public_loader_tls_object_t *objects, + size_t object_count) +{ + for (size_t index = 0; index < object_count; ++index) { + if (objects[index].module_id > KZT_GUEST_MAX_DTV_ENTRIES) { + printf_log(LOG_INFO, + "KZT Guest TLS object %p has unsupported " + "module ID %zu\n", + (void *)objects[index].link_map_addr, + objects[index].module_id); + return -1; + } + } + return 0; +} + +static int kzt_guest_tls_validate_refresh_inventory( + const kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *objects, + size_t object_count) +{ + if (kzt_guest_tls_validate_inventory(objects, object_count) != 0) { + return -1; + } + for (size_t index = 0; index < object_count; ++index) { + int found = 0; + + if (objects[index].module_id) { + continue; + } + for (size_t previous_index = 0; + previous_index < state->tls_inventory_count; + ++previous_index) { + if (kzt_guest_tls_same_object( + &objects[index], + &state->tls_inventory[previous_index])) { + found = 1; + break; + } + } + if (!found) { + printf_log(LOG_INFO, + "KZT Guest TLS cannot identify late object %p\n", + (void *)objects[index].link_map_addr); + return -1; + } + } + return 0; +} + +static int kzt_guest_tls_initialize_static_module( + const kzt_public_loader_tls_object_t *object, + const kzt_guest_dtv_entry_t *entry) +{ + void *destination; + void *image = NULL; + + if (!object || !entry || !entry->value || + entry->value == UINTPTR_MAX || !object->memory_size) { + return -1; + } + if (object->file_size) { + image = g_malloc(object->file_size); + if (kzt_materialize_guest_tls_image( + object, image, object->file_size) != 0) { + g_free(image); + return -1; + } + } + destination = lock_user( + VERIFY_WRITE, (abi_ulong)entry->value, + object->memory_size, 0); + if (!destination) { + g_free(image); + return -1; + } + memset(destination, 0, object->memory_size); + if (image) { + memcpy(destination, image, object->file_size); + g_free(image); + } + unlock_user(destination, (abi_ulong)entry->value, + object->memory_size); + return 0; +} + +static int kzt_guest_tls_has_object( + const kzt_public_loader_tls_object_t *objects, + size_t object_count, + const kzt_public_loader_tls_object_t *object) +{ + for (size_t index = 0; index < object_count; ++index) { + if (kzt_guest_tls_same_object_ignoring_module( + &objects[index], object)) { + return 1; + } + } + return 0; +} + +static int kzt_guest_tls_static_address( + const kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *object, + uintptr_t *address) +{ + uintptr_t magnitude; + + if (!state || !object || !address || + !object->static_tls_offset_valid || + object->static_tls_offset >= 0 || + object->static_tls_offset == INTPTR_MIN) { + return -1; + } + magnitude = (uintptr_t)-object->static_tls_offset; + if (state->thread_pointer < magnitude) { + return -1; + } + *address = state->thread_pointer - magnitude; + if (state->static_start > UINTPTR_MAX - state->static_size || + *address < state->static_start || + object->memory_size > state->static_size || + *address > state->static_start + state->static_size - + object->memory_size) { + return -1; + } + return 0; +} + +static int kzt_guest_tls_validate_static_offset( + const kzt_public_loader_tls_object_t *object) +{ + if (!object->static_tls_offset_needs_validation) { + return 0; + } + printf_log(LOG_INFO, + "KZT Guest TLS cannot prove ownership of static " + "offset for object %p\n", + (void *)object->link_map_addr); + return -1; +} + +static int kzt_guest_tls_lock_target( + kzt_guest_tls_state_t *target, + const kzt_guest_tls_state_t *source, + int wait) +{ + if (target == source) { + return 1; + } + for (int attempt = 0; + attempt < (wait ? KZT_GUEST_TLS_TARGET_WAIT_RETRIES : 1); + ++attempt) { + if (g_rec_mutex_trylock(&target->execution_lock)) { + return 1; + } + if (wait) { + g_usleep(1000); + } + } + return 0; +} + +static void kzt_guest_tls_unlock_target( + kzt_guest_tls_state_t *target, + const kzt_guest_tls_state_t *source) +{ + if (target != source) { + g_rec_mutex_unlock(&target->execution_lock); + } +} + +static int kzt_guest_tls_initialize_pending_static( + kzt_guest_tls_state_t *source, + const kzt_public_loader_tls_object_t *objects, + size_t object_count, + int complete_inventory) +{ + if (complete_inventory) { + size_t retained = 0; + + for (size_t index = 0; + index < source->preinitialized_static_count; ++index) { + if (kzt_guest_tls_has_object( + objects, object_count, + &source->preinitialized_static_objects[index])) { + source->preinitialized_static_objects[retained++] = + source->preinitialized_static_objects[index]; + } + } + source->preinitialized_static_count = retained; + } + + for (size_t index = 0; index < object_count; ++index) { + const kzt_public_loader_tls_object_t *object = &objects[index]; + kzt_guest_dtv_entry_t entry; + uintptr_t address; + + if (!object->static_tls_offset_valid || + kzt_guest_tls_has_object( + source->tls_inventory, source->tls_inventory_count, + object) || + kzt_guest_tls_has_object( + source->preinitialized_static_objects, + source->preinitialized_static_count, object)) { + continue; + } + if (kzt_guest_tls_validate_static_offset(object) != 0) { + return -1; + } + if (source->preinitialized_static_count == + KZT_PUBLIC_LOADER_MAX_OBJECTS || + kzt_guest_tls_static_address( + source, object, &address) != 0) { + return -1; + } + entry = (kzt_guest_dtv_entry_t) { + .value = address, + .to_free = 0, + }; + if (kzt_guest_tls_initialize_static_module( + object, &entry) != 0) { + return -1; + } + if (object->module_id) { + source->dtv[object->module_id] = entry; + } + /* Loader generation is updated only after complete reconciliation. */ + source->preinitialized_static_objects[ + source->preinitialized_static_count++] = *object; + } + return 0; +} + +static int kzt_guest_tls_adopt_dynamic( + kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *object, + kzt_guest_dtv_entry_t *entry) +{ + int dynamic_index = kzt_guest_tls_find_dynamic_index( + state, object->module_id); + + if (!entry->to_free) { + return -1; + } + if (dynamic_index >= 0 && + (state->dynamic_allocations[dynamic_index] != + (void *)entry->to_free || + !kzt_guest_tls_same_object( + &state->dynamic_objects[dynamic_index], object))) { + kzt_guest_tls_free_dynamic(state, dynamic_index); + } + if (dynamic_index < 0 || + !state->dynamic_allocations[dynamic_index]) { + if (dynamic_index < 0) { + for (size_t candidate = 0; + candidate < state->dynamic_module_count; ++candidate) { + if (!state->dynamic_allocations[candidate]) { + dynamic_index = (int)candidate; + break; + } + } + } + if (dynamic_index < 0) { + if (state->dynamic_module_count == + KZT_PUBLIC_LOADER_MAX_OBJECTS) { + return -1; + } + dynamic_index = (int)state->dynamic_module_count++; + } + } + state->dynamic_allocations[dynamic_index] = + (void *)entry->to_free; + state->dynamic_module_ids[dynamic_index] = object->module_id; + state->dynamic_objects[dynamic_index] = *object; + state->dynamic_guest_owned[dynamic_index] = 1; + entry->to_free = 0; + return 0; +} + +static void kzt_guest_tls_remember_inventory( + kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *objects, + size_t object_count) +{ + memcpy(state->tls_inventory, objects, + object_count * sizeof(*objects)); + state->tls_inventory_count = object_count; +} + +static int kzt_guest_tls_populate_dynamic_object( + kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *object) +{ + kzt_guest_dtv_entry_t *entry; + void *allocation; + void *tls_address; + int dynamic_index; + size_t alignment; + size_t allocation_size; + size_t address_offset; + + if (!object->module_id || + object->module_id > KZT_GUEST_MAX_DTV_ENTRIES) { + return -1; + } + entry = &state->dtv[object->module_id]; + if (entry->to_free) { + return kzt_guest_tls_adopt_dynamic(state, object, entry); + } + if (entry->value != UINTPTR_MAX) { + return 0; + } + dynamic_index = kzt_guest_tls_find_dynamic_index( + state, object->module_id); + allocation = dynamic_index >= 0 + ? state->dynamic_allocations[dynamic_index] + : NULL; + if (allocation && + !kzt_guest_tls_same_object( + &state->dynamic_objects[dynamic_index], object)) { + kzt_guest_tls_free_dynamic(state, dynamic_index); + allocation = NULL; + } + alignment = object->alignment < sizeof(void *) + ? sizeof(void *) : object->alignment; + if (!allocation) { + if (object->memory_size > SIZE_MAX - (alignment - 1)) { + return -1; + } + allocation_size = object->memory_size + alignment - 1; + allocation = g_try_malloc(allocation_size); + if (!allocation) { + return -1; + } + address_offset = + (object->first_byte_offset - + ((uintptr_t)allocation & (alignment - 1))) & + (alignment - 1); + tls_address = (unsigned char *)allocation + address_offset; + memset(tls_address, 0, object->memory_size); + if (object->file_size && + kzt_materialize_guest_tls_image( + object, tls_address, object->file_size) != 0) { + g_free(allocation); + return -1; + } + if (dynamic_index < 0) { + for (size_t candidate = 0; + candidate < state->dynamic_module_count; + ++candidate) { + if (!state->dynamic_allocations[candidate]) { + dynamic_index = (int)candidate; + break; + } + } + if (dynamic_index < 0) { + if (state->dynamic_module_count == + KZT_PUBLIC_LOADER_MAX_OBJECTS) { + g_free(allocation); + return -1; + } + dynamic_index = (int)state->dynamic_module_count++; + } + } + state->dynamic_allocations[dynamic_index] = allocation; + state->dynamic_module_ids[dynamic_index] = object->module_id; + state->dynamic_objects[dynamic_index] = *object; + state->dynamic_guest_owned[dynamic_index] = 0; + } else { + address_offset = + (object->first_byte_offset - + ((uintptr_t)allocation & (alignment - 1))) & + (alignment - 1); + tls_address = (unsigned char *)allocation + address_offset; + } + entry->value = (uintptr_t)tls_address; + entry->to_free = 0; + return 0; +} + +static int kzt_guest_tls_populate_dynamic( + kzt_guest_tls_state_t *state, + const kzt_public_loader_tls_object_t *tls_objects, + size_t tls_object_count) +{ + size_t dynamic_slot_count = 0; + size_t known_dynamic_count = 0; + + for (size_t previous_index = 0; + previous_index < state->tls_inventory_count; + ++previous_index) { + const kzt_public_loader_tls_object_t *previous = + &state->tls_inventory[previous_index]; + + if (previous->module_id && + previous->module_id <= KZT_GUEST_MAX_DTV_ENTRIES && + !kzt_guest_tls_has_object( + tls_objects, tls_object_count, previous)) { + state->dtv[previous->module_id].value = 0; + state->dtv[previous->module_id].to_free = 0; + } + } + kzt_guest_tls_retire_dynamic( + state, tls_objects, tls_object_count); + for (size_t index = 0; index < tls_object_count; ++index) { + size_t module_id = tls_objects[index].module_id; + + if (module_id && module_id <= KZT_GUEST_MAX_DTV_ENTRIES && + state->dtv[module_id].value == 0) { + state->dtv[module_id].value = UINTPTR_MAX; + } + } + for (size_t index = 1; index <= KZT_GUEST_MAX_DTV_ENTRIES; ++index) { + if (state->dtv[index].value == UINTPTR_MAX) { + int live = 0; + + for (size_t object_index = 0; + object_index < tls_object_count; ++object_index) { + if (tls_objects[object_index].module_id == index) { + live = 1; + break; + } + } + if (live) { + ++dynamic_slot_count; + } else { + state->dtv[index].value = 0; + state->dtv[index].to_free = 0; + } + } + } + for (size_t index = 0; index < tls_object_count; ++index) { + const kzt_public_loader_tls_object_t *object = &tls_objects[index]; + + if (object->module_id && + object->module_id <= KZT_GUEST_MAX_DTV_ENTRIES && + state->dtv[object->module_id].value == UINTPTR_MAX) { + ++known_dynamic_count; + } + } + if (known_dynamic_count != dynamic_slot_count) { + return -1; + } + + for (size_t index = 0; index < tls_object_count; ++index) { + const kzt_public_loader_tls_object_t *object = &tls_objects[index]; + + if (!object->module_id || + object->module_id > KZT_GUEST_MAX_DTV_ENTRIES || + (state->dtv[object->module_id].value != UINTPTR_MAX && + !state->dtv[object->module_id].to_free)) { + continue; + } + if (kzt_guest_tls_populate_dynamic_object( + state, object) != 0) { + return -1; + } + } + return 0; +} + +static int kzt_guest_tls_propagate_inventory( + kzt_guest_tls_state_t *source, + const kzt_public_loader_tls_object_t *tls_objects, + size_t tls_object_count, + uintptr_t generation, uintptr_t loader_generation) +{ + kzt_guest_tls_state_t + *targets[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + size_t target_count = 0; + int result = 0; + int deferred = 0; + + g_mutex_lock(&kzt_guest_tls_states_lock); + for (kzt_guest_tls_state_t *target = kzt_guest_tls_states; + target; target = target->next) { + if (target == source || + g_atomic_int_get(&target->destroying)) { + continue; + } + if (target_count == KZT_PUBLIC_LOADER_MAX_OBJECTS) { + result = -1; + break; + } + g_atomic_int_inc(&target->propagation_refs); + targets[target_count++] = target; + } + g_mutex_unlock(&kzt_guest_tls_states_lock); + + for (size_t index = 0; index < target_count; ++index) { + kzt_guest_tls_state_t *target = targets[index]; + + if (result != 0) { + g_atomic_int_dec_and_test(&target->propagation_refs); + continue; + } + if (!kzt_guest_tls_lock_target(target, source, 1)) { + /* The target will apply the pending generation on next entry. */ + deferred = 1; + g_atomic_int_dec_and_test(&target->propagation_refs); + continue; + } + if (target->inventory_generation >= generation) { + kzt_guest_tls_unlock_target(target, source); + g_atomic_int_dec_and_test(&target->propagation_refs); + continue; + } + mmap_lock(); + if (kzt_guest_tls_initialize_pending_static( + target, tls_objects, tls_object_count, 1) != 0 || + kzt_guest_tls_populate_dynamic( + target, tls_objects, tls_object_count) != 0) { + fprintf(stderr, + "KZT Guest TLS propagation could not initialize " + "target %p\n", (void *)target); + result = -1; + } else { + target->dtv[0].value = MAX(target->dtv[0].value, + loader_generation); + target->inventory_generation = generation; + kzt_guest_tls_remember_inventory( + target, tls_objects, tls_object_count); + } + mmap_unlock(); + kzt_guest_tls_unlock_target(target, source); + g_atomic_int_dec_and_test(&target->propagation_refs); + } + return result != 0 ? result : deferred; +} + +static int kzt_guest_tls_clone_static(CPUX86State *parent, + CPUX86State *child) +{ + kzt_public_loader_tls_object_t + tls_objects[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + kzt_guest_tls_state_t *state = NULL; + kzt_guest_parent_tls_snapshot_t *parent_snapshot; + kzt_guest_dtv_entry_t *child_dtv; + kzt_x86_64_tcbhead_t *parent_tcb; + kzt_x86_64_tcbhead_t *child_tcb; + TaskState *child_ts; + uintptr_t aligned_static; + uintptr_t child_tp; + size_t static_size; + size_t static_align; + size_t tcb_size; + size_t tls_object_count = 0; + size_t index; + int collect_result; + + if (!parent || !child || !parent->segs[R_FS].base || + !child->kzt_guest_tls_parent_snapshot) { + return -1; + } + parent_snapshot = child->kzt_guest_tls_parent_snapshot; + if (parent_snapshot->parent_tp != parent->segs[R_FS].base) { + return -1; + } + collect_result = kzt_collect_guest_tls_objects( + tls_objects, KZT_PUBLIC_LOADER_MAX_OBJECTS, &tls_object_count); + if (collect_result != 0) { + /* No Guest allocation or helper call has taken place yet. */ + return collect_result; + } + if (kzt_guest_tls_validate_inventory( + tls_objects, tls_object_count) != 0) { + return -1; + } + if (kzt_guest_tls_resolve_loader() != 0) { + return -1; + } + static_size = 0; + static_align = 0; + RunFunctionWithStateInternal(guest_get_tls_static_info, 2, + (uint64_t)(uintptr_t)&static_size, + (uint64_t)(uintptr_t)&static_align); + if (!static_size || static_size > KZT_GUEST_MAX_STATIC_TLS_SIZE || + !static_align || (static_align & (static_align - 1)) || + static_align > KZT_GUEST_MAX_TCB_SIZE) { + return -1; + } + tcb_size = kzt_guest_tls_find_tcb_size( + parent_snapshot, static_size, static_align); + if (!tcb_size || tcb_size > static_size) { + return -1; + } + + state = g_new0(kzt_guest_tls_state_t, 1); + g_rec_mutex_init(&state->execution_lock); + state->static_allocation = g_malloc( + static_size + static_align - 1); + aligned_static = ((uintptr_t)state->static_allocation + + static_align - 1) & + ~(uintptr_t)(static_align - 1); + memset((void *)aligned_static, 0, static_size); + child_tp = aligned_static + static_size - tcb_size; + state->static_start = aligned_static; + state->static_size = static_size; + state->thread_pointer = child_tp; + + state->dtv_allocation = g_malloc0( + (KZT_GUEST_MAX_DTV_ENTRIES + 2) * + sizeof(kzt_guest_dtv_entry_t)); + child_dtv = state->dtv_allocation; + child_dtv[0].value = KZT_GUEST_MAX_DTV_ENTRIES; + ++child_dtv; + state->dtv = child_dtv; + + parent_tcb = (kzt_x86_64_tcbhead_t *)parent_snapshot->tcb_bytes; + child_tcb = (kzt_x86_64_tcbhead_t *)child_tp; + child_ts = env_cpu(child)->opaque; + child_tcb->tcb = child_tp; + child_tcb->dtv = child_dtv; + child_tcb->self = child_tp; + child_tcb->multiple_threads = 1; + child_tcb->sysinfo = parent_tcb->sysinfo; + child_tcb->stack_guard = parent_tcb->stack_guard; + child_tcb->pointer_guard = parent_tcb->pointer_guard; + child_tcb->feature_1 = parent_tcb->feature_1; + + if (!child_ts || child_ts->ts_tid <= 0 || + parent_snapshot->tid_offset > + tcb_size - sizeof(uint32_t) || + parent_snapshot->robust_head_offset > + tcb_size - sizeof(struct robust_list_head) || + parent_snapshot->robust_prev_offset > + tcb_size - sizeof(uintptr_t)) { + goto fail; + } + { + struct robust_list_head *robust_head = + (struct robust_list_head *)(child_tp + + parent_snapshot->robust_head_offset); + uintptr_t *robust_prev = (uintptr_t *)(child_tp + + parent_snapshot->robust_prev_offset); + uint32_t *tid = (uint32_t *)(child_tp + + parent_snapshot->tid_offset); + + *tid = (uint32_t)child_ts->ts_tid; + robust_head->list.next = &robust_head->list; + robust_head->futex_offset = + parent_snapshot->robust_futex_offset; + robust_head->list_op_pending = NULL; + *robust_prev = (uintptr_t)robust_head; + state->robust_head_addr = (uintptr_t)robust_head; + state->robust_prev_addr = (uintptr_t)robust_prev; + state->robust_futex_offset = + parent_snapshot->robust_futex_offset; + } + /* + * Guest loader helpers may acquire pthread recursive locks. A zero + * TID would alias the unlocked owner value and bypass mutual exclusion. + * Publish a complete thread descriptor before executing Guest code. + */ + child->segs[R_FS].base = child_tp; + child->kzt_guest_tls_allocation = state; + state->refreshing = 1; + if (RunFunctionWithStateInternal( + guest_allocate_tls_init, 2, + (uint64_t)child_tp, (uint64_t)1) != child_tp) { + goto fail; + } + if (child_tcb->dtv != child_dtv) { + goto fail; + } + if (kzt_guest_tls_resolve_new_module_ids( + state, tls_objects, tls_object_count, 0) != 0) { + goto fail; + } + if (kzt_guest_tls_populate_dynamic( + state, tls_objects, tls_object_count) != 0) { + goto fail; + } + kzt_guest_tls_remember_inventory( + state, tls_objects, tls_object_count); + memcpy(kzt_guest_tls_loader_inventory, tls_objects, + tls_object_count * sizeof(*tls_objects)); + kzt_guest_tls_loader_inventory_count = tls_object_count; + qatomic_set(&kzt_guest_tls_loader_generation, + state->dtv[0].value); + kzt_guest_tls_loader_inventory_initialized = 1; + if (!kzt_guest_tls_process_inventory_initialized) { + qatomic_set(&kzt_guest_tls_inventory_generation, 1); + memcpy(kzt_guest_tls_process_inventory, tls_objects, + tls_object_count * sizeof(*tls_objects)); + kzt_guest_tls_process_inventory_count = tls_object_count; + kzt_guest_tls_process_inventory_initialized = 1; + } else { + int changed = kzt_guest_tls_update_process_inventory( + tls_objects, tls_object_count); + + if (changed < 0) { + goto fail; + } + if (changed) { + qatomic_set(&kzt_guest_tls_pending_propagation_generation, + qatomic_read(&kzt_guest_tls_inventory_generation)); + } + } + state->inventory_generation = qatomic_read(&kzt_guest_tls_inventory_generation); + state->refreshing = 0; + return 0; + +fail: + if (state) { + for (index = 0; index < state->dynamic_module_count; ++index) { + kzt_guest_tls_free_dynamic(state, index); + } + kzt_guest_tls_release_loader_state(state); + g_free(state->static_allocation); + g_rec_mutex_clear(&state->execution_lock); + g_free(state); + } + child->kzt_guest_tls_allocation = NULL; + return -1; +} + +static int kzt_guest_tls_refresh_internal(CPUX86State *env, + int allow_propagation, + int commit_inventory) +{ + kzt_public_loader_tls_object_t + tls_objects[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + kzt_guest_tls_state_t *state; + kzt_x86_64_tcbhead_t *tcb; + size_t tls_object_count = 0; + int inventory_changed; + int inventory_has_additions; + int loader_inventory_has_additions = 0; + int process_inventory_changed; + int propagate_inventory = 0; + uintptr_t propagation_generation = 0; + uintptr_t propagation_loader_generation = 0; + int result = -1; + int collect_result; + const char *failure_stage = "collect"; + + if (!env || !env->kzt_guest_tls_allocation) { + return 0; + } + state = env->kzt_guest_tls_allocation; + g_rec_mutex_lock(&state->execution_lock); + if (state->refreshing) { + g_rec_mutex_unlock(&state->execution_lock); + return 0; + } + mmap_lock(); + g_mutex_lock(&kzt_guest_tls_states_lock); + if (state->refreshing) { + result = 0; + goto unlock; + } + collect_result = kzt_collect_guest_tls_objects( + tls_objects, KZT_PUBLIC_LOADER_MAX_OBJECTS, + &tls_object_count); + if (collect_result != 0) { + if (collect_result == KZT_GUEST_TLS_REFRESH_BUSY) { + result = KZT_GUEST_TLS_REFRESH_BUSY; + } + goto unlock; + } + if (kzt_guest_tls_resolve_new_module_ids( + state, tls_objects, tls_object_count, 0) != 0) { + failure_stage = "module-id"; + goto unlock; + } + if (kzt_guest_tls_initialize_pending_static( + state, tls_objects, tls_object_count, 1) != 0) { + failure_stage = "static-image"; + goto unlock; + } + if (kzt_guest_tls_validate_refresh_inventory( + state, tls_objects, tls_object_count) != 0) { + failure_stage = "inventory-validation"; + goto unlock; + } + inventory_changed = !kzt_guest_tls_inventory_matches( + state, tls_objects, tls_object_count); + inventory_has_additions = inventory_changed && + kzt_guest_tls_inventory_has_additions( + state, tls_objects, tls_object_count, + &loader_inventory_has_additions); + state->refreshing = 1; + tcb = (kzt_x86_64_tcbhead_t *)env->segs[R_FS].base; + if (!tcb || tcb->dtv != state->dtv) { + failure_stage = "dtv-ownership"; + goto out; + } + if (kzt_guest_tls_populate_dynamic( + state, tls_objects, tls_object_count) != 0) { + failure_stage = "dynamic-image"; + goto out; + } + if (!commit_inventory) { + result = 0; + goto out; + } + if (loader_inventory_has_additions) { + uintptr_t generation = qatomic_read( + &kzt_guest_tls_loader_generation); + + if (!generation) { + failure_stage = "loader-generation-snapshot"; + goto out; + } + state->dtv[0].value = MAX(state->dtv[0].value, generation); + } + process_inventory_changed = kzt_guest_tls_update_process_inventory( + tls_objects, tls_object_count); + if (process_inventory_changed < 0) { + failure_stage = "process-generation"; + goto out; + } + if (inventory_has_additions && process_inventory_changed) { + qatomic_set(&kzt_guest_tls_pending_propagation_generation, + qatomic_read(&kzt_guest_tls_inventory_generation)); + } + state->inventory_generation = qatomic_read(&kzt_guest_tls_inventory_generation); + propagate_inventory = allow_propagation && + qatomic_read(&kzt_guest_tls_pending_propagation_generation) != 0; + propagation_generation = state->inventory_generation; + propagation_loader_generation = state->dtv[0].value; + if (inventory_changed) { + kzt_guest_tls_remember_inventory( + state, tls_objects, tls_object_count); + } + result = 0; + +out: + state->refreshing = 0; +unlock: + g_mutex_unlock(&kzt_guest_tls_states_lock); + mmap_unlock(); + g_rec_mutex_unlock(&state->execution_lock); + if (result == 0 && propagate_inventory) { + int propagation_result = kzt_guest_tls_propagate_inventory( + state, tls_objects, tls_object_count, + propagation_generation, propagation_loader_generation); + + if (propagation_result < 0) { + failure_stage = "thread-propagation"; + result = -1; + } else if (propagation_result == 0) { + g_mutex_lock(&kzt_guest_tls_states_lock); + if (qatomic_read( + &kzt_guest_tls_pending_propagation_generation) == + propagation_generation) { + qatomic_set( + &kzt_guest_tls_pending_propagation_generation, 0); + } + g_mutex_unlock(&kzt_guest_tls_states_lock); + } + } + if (result != 0 && result != KZT_GUEST_TLS_REFRESH_BUSY) { + fprintf(stderr, + "KZT Guest TLS refresh failed at %s for state %p\n", + failure_stage, (void *)state); + } + if (result == 0 && commit_inventory) { + kzt_guest_tls_publish_stable_epoch(); + } + return result; +} + +int kzt_guest_tls_refresh(CPUX86State *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return 0; + } + int result; + + kzt_guest_tls_initialize_refresh_lock(); + g_rec_mutex_lock(&kzt_guest_tls_refresh_lock); + result = kzt_guest_tls_refresh_internal(env, 1, 1); + g_rec_mutex_unlock(&kzt_guest_tls_refresh_lock); + return result; +} + +int kzt_guest_tls_refresh_if_needed(CPUX86State *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return 0; + } + kzt_guest_tls_state_t *state; + int result; + + if (!env || !env->kzt_guest_tls_allocation) { + return 0; + } + state = env->kzt_guest_tls_allocation; + if (kzt_guest_loader_operation_active()) { + return kzt_guest_tls_refresh_local(env); + } + g_rec_mutex_lock(&state->execution_lock); + if (state->refreshing || + kzt_guest_tls_can_reuse_snapshot(state)) { + g_rec_mutex_unlock(&state->execution_lock); + return 0; + } + g_rec_mutex_unlock(&state->execution_lock); + + /* Slow refresh has one writer. Waiters must not pin their state lock. */ + kzt_guest_tls_initialize_refresh_lock(); + g_rec_mutex_lock(&kzt_guest_tls_refresh_lock); + g_rec_mutex_lock(&state->execution_lock); + if (state->refreshing || + kzt_guest_tls_can_reuse_snapshot(state)) { + result = 0; + } else { + result = kzt_guest_tls_refresh_internal(env, 1, 1); + } + g_rec_mutex_unlock(&state->execution_lock); + g_rec_mutex_unlock(&kzt_guest_tls_refresh_lock); + return result; +} + +int kzt_guest_tls_refresh_local(CPUX86State *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return 0; + } + int result; + + kzt_guest_tls_initialize_refresh_lock(); + g_rec_mutex_lock(&kzt_guest_tls_refresh_lock); + result = kzt_guest_tls_refresh_internal(env, 0, 0); + g_rec_mutex_unlock(&kzt_guest_tls_refresh_lock); + return result; +} + +int kzt_guest_tls_preinitialize_static(CPUX86State *env, + uintptr_t link_map_addr) +{ + kzt_public_loader_tls_object_t tls_object; + kzt_guest_tls_state_t *state; + int has_tls = 0; + int result = -1; + + if (!env || !env->kzt_guest_tls_allocation || !link_map_addr) { + return 0; + } + state = env->kzt_guest_tls_allocation; + g_rec_mutex_lock(&state->execution_lock); + mmap_lock(); + g_mutex_lock(&kzt_guest_tls_states_lock); + result = kzt_collect_guest_tls_object( + link_map_addr, &tls_object, &has_tls); + if (result != 0) { + goto out; + } + if (!has_tls) { + result = 0; + goto out; + } + result = kzt_guest_tls_resolve_new_module_ids( + state, &tls_object, 1, 0); + if (result != 0) { + goto out; + } + if (!kzt_guest_tls_process_inventory_initialized || + qatomic_read(&kzt_guest_tls_inventory_generation) == UINTPTR_MAX) { + goto out; + } + if (tls_object.static_tls_offset_valid) { + result = kzt_guest_tls_initialize_pending_static( + state, &tls_object, 1, 0); + } else if (!tls_object.module_id || + tls_object.module_id > KZT_GUEST_MAX_DTV_ENTRIES) { + result = -1; + } else { + kzt_guest_dtv_entry_t *entry = + &state->dtv[tls_object.module_id]; + int dynamic_index = kzt_guest_tls_find_dynamic_index( + state, tls_object.module_id); + + if (dynamic_index < 0 || + !kzt_guest_tls_same_object( + &state->dynamic_objects[dynamic_index], + &tls_object)) { + entry->value = UINTPTR_MAX; + entry->to_free = 0; + } + result = kzt_guest_tls_populate_dynamic_object( + state, &tls_object); + if (result == 0 && + kzt_guest_tls_initialize_static_module( + &tls_object, entry) != 0) { + result = -1; + } + + } +out: + g_mutex_unlock(&kzt_guest_tls_states_lock); + mmap_unlock(); + g_rec_mutex_unlock(&state->execution_lock); + return result; +} + +static int kzt_guest_tls_initialize_libc(CPUX86State *env) +{ + if (!env || !env->kzt_guest_tls_allocation) { + return -1; + } + if (!guest_uselocale) { + guest_uselocale = kzt_resolve_guest_symbol("uselocale"); + if (!guest_uselocale) { + guest_uselocale = kzt_resolve_guest_symbol("__uselocale"); + } + } + if (!guest_uselocale) { + guest_uselocale = kzt_find_guest_libc_symbol("uselocale"); + if (!guest_uselocale) { + guest_uselocale = kzt_find_guest_libc_symbol("__uselocale"); + } + } + if (!guest_uselocale) { + printf_log(LOG_INFO, + "KZT cannot resolve the Guest libc locale initializer\n"); + return -1; + } + if (!guest_ctype_init) { + guest_ctype_init = + kzt_find_guest_libc_symbol("__ctype_init"); + if (!guest_ctype_init) { + guest_ctype_init = + kzt_resolve_guest_symbol("__ctype_init"); + } + } + if (!guest_ctype_init) { + printf_log(LOG_INFO, + "KZT cannot resolve Guest __ctype_init\n"); + return -1; + } + + if (RunFunctionWithStateInternalNoRefresh( + guest_uselocale, 1, (uint64_t)-1) == 0) { + return -1; + } + (void)RunFunctionWithStateInternalNoRefresh( + guest_ctype_init, 0); + return 0; +} + +int kzt_guest_tls_initialize(CPUX86State *parent, CPUX86State *child) +{ + if (!latx_kzt_guest_tls_enabled()) { + return -1; + } + int result; + + if (!parent || !child) { + return -1; + } + kzt_guest_tls_initialize_refresh_lock(); + g_rec_mutex_lock(&kzt_guest_tls_refresh_lock); + mmap_lock(); + g_mutex_lock(&kzt_guest_tls_states_lock); + result = kzt_guest_tls_clone_static(parent, child); + kzt_guest_tls_release_parent_snapshot(child); + if (result != 0) { + g_mutex_unlock(&kzt_guest_tls_states_lock); + mmap_unlock(); + g_rec_mutex_unlock(&kzt_guest_tls_refresh_lock); + return result; + } + result = kzt_guest_tls_initialize_libc(child); + if (result == 0) { + kzt_guest_tls_state_t *state = + child->kzt_guest_tls_allocation; + + state->next = kzt_guest_tls_states; + kzt_guest_tls_states = state; + } + g_mutex_unlock(&kzt_guest_tls_states_lock); + mmap_unlock(); + if (result == 0) { + kzt_guest_tls_publish_stable_epoch(); + } + g_rec_mutex_unlock(&kzt_guest_tls_refresh_lock); + return result; +} + +void kzt_guest_tls_destroy(CPUX86State *env) +{ + kzt_guest_tls_state_t *state; + + if (!env) { + return; + } + kzt_guest_tls_release_parent_snapshot(env); + state = env->kzt_guest_tls_allocation; + if (!state) { + return; + } + if (g_atomic_int_get(&state->execution_depth) != 0 || + g_atomic_pointer_get(&state->execution_owner) != NULL) { + fprintf(stderr, + "KZT Guest TLS teardown raced active Guest execution; " + "refusing to continue\n"); + _exit(EXIT_FAILURE); + } + g_mutex_lock(&kzt_guest_tls_states_lock); + g_atomic_int_set(&state->destroying, 1); + if (kzt_guest_tls_states == state) { + kzt_guest_tls_states = state->next; + } else { + kzt_guest_tls_state_t *previous = kzt_guest_tls_states; + + while (previous && previous->next != state) { + previous = previous->next; + } + if (previous) { + previous->next = state->next; + } + } + if (!kzt_guest_tls_states) { + memset(kzt_guest_tls_process_inventory, 0, + sizeof(kzt_guest_tls_process_inventory)); + kzt_guest_tls_process_inventory_count = 0; + memset(kzt_guest_tls_loader_inventory, 0, + sizeof(kzt_guest_tls_loader_inventory)); + kzt_guest_tls_loader_inventory_count = 0; + qatomic_set(&kzt_guest_tls_loader_generation, 0); + kzt_guest_tls_loader_inventory_initialized = 0; + qatomic_set(&kzt_guest_tls_inventory_generation, 0); + qatomic_set(&kzt_guest_tls_pending_propagation_generation, 0); + kzt_guest_tls_process_inventory_initialized = 0; + } + g_mutex_unlock(&kzt_guest_tls_states_lock); + for (int attempt = 0; + g_atomic_int_get(&state->propagation_refs) != 0 && + attempt < KZT_GUEST_TLS_TARGET_WAIT_RETRIES; + ++attempt) { + g_usleep(1000); + } + if (g_atomic_int_get(&state->propagation_refs) != 0) { + fprintf(stderr, + "KZT Guest TLS state remained in propagation during " + "thread teardown; refusing to continue\n"); + _exit(EXIT_FAILURE); + } + for (size_t index = 0; index < state->dynamic_module_count; ++index) { + kzt_guest_tls_free_dynamic(state, index); + } + kzt_guest_tls_release_loader_state(state); + g_free(state->static_allocation); + g_rec_mutex_clear(&state->execution_lock); + g_free(state); + env->kzt_guest_tls_allocation = NULL; +} + +static int kzt_guest_tls_robust_futex_address( + uintptr_t list_addr, intptr_t futex_offset, + uintptr_t *futex_addr) +{ + if (!list_addr || !futex_addr || + (futex_offset > 0 && + list_addr > UINTPTR_MAX - (uintptr_t)futex_offset) || + (futex_offset < 0 && + list_addr < (uintptr_t)-futex_offset)) { + return -1; + } + *futex_addr = (uintptr_t)((intptr_t)list_addr + futex_offset); + return 0; +} + +static int kzt_guest_tls_mark_robust_owner_died( + uintptr_t list_addr, intptr_t futex_offset, uint32_t tid) +{ + uintptr_t futex_addr; + uint32_t *futex_word; + uint32_t old_value; + + if ((list_addr & 1) || + kzt_guest_tls_robust_futex_address( + list_addr, futex_offset, &futex_addr) != 0) { + return -1; + } + futex_word = lock_user( + VERIFY_WRITE, (abi_ulong)futex_addr, + sizeof(*futex_word), 0); + if (!futex_word) { + return -1; + } + old_value = qatomic_read(futex_word); + while ((old_value & FUTEX_TID_MASK) == tid) { + uint32_t new_value = + (old_value & FUTEX_WAITERS) | FUTEX_OWNER_DIED; + uint32_t observed = qatomic_cmpxchg( + futex_word, old_value, new_value); + + if (observed == old_value) { + if (old_value & FUTEX_WAITERS) { + long woken = syscall( + SYS_futex, futex_word, FUTEX_WAKE, 1, + NULL, NULL, 0); + + if (woken == 0) { + (void)syscall( + SYS_futex, futex_word, + FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1, + NULL, NULL, 0); + } + } + break; + } + old_value = observed; + } + unlock_user(futex_word, (abi_ulong)futex_addr, + sizeof(*futex_word)); + return 0; +} + +int kzt_guest_tls_cleanup_robust_list(CPUX86State *env) +{ + kzt_guest_tls_state_t *state; + CPUState *cpu; + TaskState *ts; + struct robust_list_head head; + struct robust_list_head *locked_head; + uintptr_t current; + uintptr_t pending; + size_t count = 0; + int result = 0; + + if (!env || !env->kzt_guest_tls_allocation) { + return 0; + } + state = env->kzt_guest_tls_allocation; + cpu = env_cpu(env); + ts = cpu ? cpu->opaque : NULL; + if (!state->robust_head_addr || !ts || ts->ts_tid <= 0) { + return -1; + } + locked_head = lock_user( + VERIFY_READ, (abi_ulong)state->robust_head_addr, + sizeof(head), 1); + if (!locked_head) { + return -1; + } + memcpy(&head, locked_head, sizeof(head)); + unlock_user(locked_head, (abi_ulong)state->robust_head_addr, 0); + current = (uintptr_t)head.list.next; + pending = (uintptr_t)head.list_op_pending; + for (; + current && current != state->robust_head_addr && + count < KZT_GUEST_ROBUST_LIST_LIMIT; + ++count) { + struct robust_list *entry; + uintptr_t next; + + entry = lock_user( + VERIFY_READ, (abi_ulong)(current & ~(uintptr_t)1), + sizeof(*entry), 1); + if (!entry) { + result = -1; + break; + } + next = (uintptr_t)entry->next; + unlock_user(entry, + (abi_ulong)(current & ~(uintptr_t)1), 0); + if (kzt_guest_tls_mark_robust_owner_died( + current, state->robust_futex_offset, + (uint32_t)ts->ts_tid) != 0) { + result = -1; + } + current = next; + } + if (current && current != state->robust_head_addr) { + result = -1; + } + if (pending) { + if (kzt_guest_tls_mark_robust_owner_died( + pending, state->robust_futex_offset, + (uint32_t)ts->ts_tid) != 0) { + result = -1; + } + } + locked_head = lock_user( + VERIFY_WRITE, (abi_ulong)state->robust_head_addr, + sizeof(head), 0); + if (locked_head) { + locked_head->list.next = + (struct robust_list *)state->robust_head_addr; + locked_head->list_op_pending = NULL; + unlock_user(locked_head, + (abi_ulong)state->robust_head_addr, + sizeof(head)); + } else { + result = -1; + } + if (state->robust_prev_addr) { + uintptr_t *robust_prev = lock_user( + VERIFY_WRITE, (abi_ulong)state->robust_prev_addr, + sizeof(*robust_prev), 0); + + if (robust_prev) { + *robust_prev = state->robust_head_addr; + unlock_user(robust_prev, + (abi_ulong)state->robust_prev_addr, + sizeof(*robust_prev)); + } else { + result = -1; + } + } + return result; +} diff --git a/target/i386/latx/context/kzt_public_loader_observer.c b/target/i386/latx/context/kzt_public_loader_observer.c index 6162d3852f..6d47019462 100644 --- a/target/i386/latx/context/kzt_public_loader_observer.c +++ b/target/i386/latx/context/kzt_public_loader_observer.c @@ -26,13 +26,25 @@ _Static_assert(sizeof(kzt_x86_64_link_map_prefix_t) == 40, #define KZT_X86_64_ELFCLASS64 2 #define KZT_X86_64_ELFDATA2LSB 1 #define KZT_X86_64_EV_CURRENT 1 +#define KZT_X86_64_ET_EXEC 2 +#define KZT_X86_64_ET_DYN 3 +#define KZT_X86_64_EM_X86_64 62 +#define KZT_X86_64_PT_LOAD 1 #define KZT_X86_64_PT_GNU_RELRO UINT32_C(0x6474e552) +#define KZT_X86_64_PT_TLS 7 #define KZT_X86_64_DT_HASH 4 -#define KZT_X86_64_DT_STRTAB 5 #define KZT_X86_64_DT_SYMTAB 6 +#define KZT_X86_64_DT_STRTAB 5 #define KZT_X86_64_DT_STRSZ 10 #define KZT_X86_64_DT_SYMENT 11 #define KZT_X86_64_DT_GNU_HASH INT64_C(0x6ffffef5) +#define KZT_X86_64_DT_RELA 7 +#define KZT_X86_64_DT_RELASZ 8 +#define KZT_X86_64_DT_RELAENT 9 +#define KZT_X86_64_R_DTPMOD64 16 +#define KZT_X86_64_R_TPOFF64 18 +#define KZT_X86_64_R_64 1 +#define KZT_X86_64_R_RELATIVE 8 #define KZT_PUBLIC_LOADER_MAX_DYNAMIC_ENTRIES 4096 #define KZT_PUBLIC_LOADER_MAX_SYMBOLS (1024 * 1024) #define KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME 512 @@ -74,12 +86,20 @@ typedef struct kzt_x86_64_symbol { uint64_t size; } kzt_x86_64_symbol_t; +typedef struct kzt_x86_64_relocation { + uint64_t offset; + uint64_t info; + int64_t addend; +} kzt_x86_64_relocation_t; + _Static_assert(sizeof(kzt_x86_64_elf_header_t) == 64, "unexpected x86_64 ELF header layout"); _Static_assert(sizeof(kzt_x86_64_program_header_t) == 56, "unexpected x86_64 program header layout"); _Static_assert(sizeof(kzt_x86_64_symbol_t) == 24, "unexpected x86_64 symbol layout"); +_Static_assert(sizeof(kzt_x86_64_relocation_t) == 24, + "unexpected x86_64 relocation layout"); static int kzt_public_loader_read( const kzt_public_loader_reader_t *reader, @@ -243,6 +263,53 @@ static int kzt_public_loader_symbol_name_matches( return 1; } +static kzt_public_loader_result_t +kzt_public_loader_validate_symbol_name( + const kzt_public_loader_reader_t *reader, + uintptr_t name_addr, + size_t available) +{ + size_t limit = available < KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME + 1 + ? available + : KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME + 1; + + for (size_t index = 0; index < limit; ++index) { + char byte; + + if (name_addr > UINTPTR_MAX - index || + kzt_public_loader_read( + reader, name_addr + index, &byte, sizeof(byte)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (!byte) { + return index ? KZT_PUBLIC_LOADER_OK + : KZT_PUBLIC_LOADER_INVALID_STATE; + } + } + return KZT_PUBLIC_LOADER_LIMIT; +} + +static kzt_public_loader_result_t kzt_public_loader_copy_symbol_name( + const kzt_public_loader_reader_t *reader, + uintptr_t name_addr, + char name[KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME + 1]) +{ + for (size_t index = 0; + index <= KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME; ++index) { + if (name_addr > UINTPTR_MAX - index || + kzt_public_loader_read( + reader, name_addr + index, + &name[index], sizeof(name[index])) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (!name[index]) { + return index ? KZT_PUBLIC_LOADER_OK + : KZT_PUBLIC_LOADER_INVALID_STATE; + } + } + return KZT_PUBLIC_LOADER_LIMIT; +} + static kzt_public_loader_result_t kzt_public_loader_find_object_symbol( const kzt_public_loader_object_t *object, const kzt_public_loader_reader_t *reader, @@ -328,7 +395,8 @@ static kzt_public_loader_result_t kzt_public_loader_find_object_symbol( } if (hash_value) { if (kzt_public_loader_read(reader, hash_addr, - hash_header, sizeof(hash_header)) != 0) { + hash_header, + sizeof(hash_header)) != 0) { return KZT_PUBLIC_LOADER_READ_ERROR; } if (!hash_header[1]) { @@ -364,7 +432,8 @@ static kzt_public_loader_result_t kzt_public_loader_find_object_symbol( return KZT_PUBLIC_LOADER_READ_ERROR; } if (!symbol.name || symbol.name >= string_table_size || - !symbol.section_index || !symbol.value) { + !symbol.section_index || + (!symbol.value && (symbol.info & 0xf) != 6)) { continue; } if (string_table_addr > UINTPTR_MAX - symbol.name) { @@ -389,108 +458,279 @@ static kzt_public_loader_result_t kzt_public_loader_find_object_symbol( return KZT_PUBLIC_LOADER_NOT_FOUND; } -static kzt_public_loader_result_t kzt_public_loader_find_r_debug( - uintptr_t dynamic_addr, - size_t max_dynamic_entries, +kzt_public_loader_result_t kzt_public_loader_find_symbol_in_object( + const kzt_public_loader_object_t *object, const kzt_public_loader_reader_t *reader, - uintptr_t *r_debug_addr) + const char *symbol_name, + uintptr_t *symbol_addr) { - size_t index; + size_t symbol_name_size; - if (!dynamic_addr || !max_dynamic_entries || !reader || - !reader->read_memory || !r_debug_addr) { + if (!object || !reader || !reader->read_memory || + !symbol_name || !symbol_addr) { return KZT_PUBLIC_LOADER_INVALID_INPUT; } - *r_debug_addr = 0; + symbol_name_size = strlen(symbol_name); + if (!symbol_name_size || + symbol_name_size > KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + *symbol_addr = 0; + return kzt_public_loader_find_object_symbol( + object, reader, symbol_name, symbol_name_size, symbol_addr); +} - for (index = 0; index < max_dynamic_entries; ++index) { +static kzt_public_loader_result_t kzt_public_loader_tls_relocations( + const kzt_public_loader_object_t *object, + const kzt_public_loader_reader_t *reader, + kzt_public_loader_tls_object_t *tls_object) +{ + uint64_t rela_value = 0; + uint64_t rela_size = 0; + uint64_t rela_entry_size = 0; + uint64_t symbol_table_value = 0; + uint64_t symbol_entry_size = 0; + uint64_t string_table_value = 0; + uint64_t string_table_size = 0; + uintptr_t rela_addr; + uintptr_t symbol_table_addr = 0; + uintptr_t string_table_addr = 0; + size_t index; + int terminated = 0; + + tls_object->module_id = 0; + tls_object->static_tls_offset = 0; + tls_object->static_tls_offset_valid = 0; + tls_object->static_tls_offset_needs_validation = 0; + tls_object->static_tls_offset_pending = 0; + tls_object->static_tls_symbol_value = 0; + tls_object->static_tls_symbol_name_addr = 0; + if (!object->dynamic_addr) { + return KZT_PUBLIC_LOADER_OK; + } + for (index = 0; index < KZT_PUBLIC_LOADER_MAX_DYNAMIC_ENTRIES; + ++index) { kzt_x86_64_dynamic_entry_t entry; uintptr_t entry_addr; if (kzt_public_loader_add_offset( - dynamic_addr, index, sizeof(entry), &entry_addr) != 0) { + object->dynamic_addr, index, sizeof(entry), + &entry_addr) != 0) { return KZT_PUBLIC_LOADER_OVERFLOW; } - if (kzt_public_loader_read( - reader, entry_addr, &entry, sizeof(entry)) != 0) { + if (kzt_public_loader_read(reader, entry_addr, + &entry, sizeof(entry)) != 0) { return KZT_PUBLIC_LOADER_READ_ERROR; } - if (entry.tag == KZT_X86_64_DT_DEBUG && entry.value) { - *r_debug_addr = (uintptr_t)entry.value; - return KZT_PUBLIC_LOADER_OK; - } if (entry.tag == KZT_X86_64_DT_NULL) { - return KZT_PUBLIC_LOADER_NOT_FOUND; + terminated = 1; + break; } - } - return KZT_PUBLIC_LOADER_NOT_FOUND; -} - -static int kzt_public_loader_contains(const uintptr_t *maps, - size_t map_count, - uintptr_t map_addr) -{ - size_t index; - - for (index = 0; index < map_count; ++index) { - if (maps[index] == map_addr) { - return 1; + switch (entry.tag) { + case KZT_X86_64_DT_RELA: + rela_value = entry.value; + break; + case KZT_X86_64_DT_RELASZ: + rela_size = entry.value; + break; + case KZT_X86_64_DT_RELAENT: + rela_entry_size = entry.value; + break; + case KZT_X86_64_DT_SYMTAB: + symbol_table_value = entry.value; + break; + case KZT_X86_64_DT_SYMENT: + symbol_entry_size = entry.value; + break; + case KZT_X86_64_DT_STRTAB: + string_table_value = entry.value; + break; + case KZT_X86_64_DT_STRSZ: + string_table_size = entry.value; + break; + default: + break; } } - return 0; -} - -static int kzt_public_loader_objects_contain( - const kzt_public_loader_object_t *objects, - size_t object_count, - uintptr_t map_addr) -{ - size_t index; + if (!terminated) { + return KZT_PUBLIC_LOADER_LIMIT; + } + if (!rela_value || !rela_size || + rela_entry_size != sizeof(kzt_x86_64_relocation_t)) { + return KZT_PUBLIC_LOADER_OK; + } + if (rela_size % rela_entry_size || + rela_size / rela_entry_size > KZT_PUBLIC_LOADER_MAX_SYMBOLS) { + return KZT_PUBLIC_LOADER_LIMIT; + } + if (kzt_public_loader_dynamic_pointer( + object, rela_value, &rela_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + if (symbol_table_value && + (symbol_entry_size != sizeof(kzt_x86_64_symbol_t) || + kzt_public_loader_dynamic_pointer( + object, symbol_table_value, &symbol_table_addr) != 0)) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (string_table_value && + kzt_public_loader_dynamic_pointer( + object, string_table_value, &string_table_addr) != 0) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + for (index = 0; index < rela_size / rela_entry_size; ++index) { + kzt_x86_64_relocation_t relocation; + kzt_x86_64_symbol_t symbol; + uintptr_t entry_addr; + uintptr_t target_addr; + uint64_t relocated_module_id; + uint32_t symbol_index; + uint32_t relocation_type; + int ownership_proven; + int validation_required = 0; - for (index = 0; index < object_count; ++index) { - if (objects[index].link_map_addr == map_addr) { - return 1; + if (kzt_public_loader_add_offset( + rela_addr, index, sizeof(relocation), + &entry_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; } - } - return 0; -} + if (kzt_public_loader_read(reader, entry_addr, + &relocation, sizeof(relocation)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + relocation_type = (uint32_t)relocation.info; + if (relocation_type != KZT_X86_64_R_DTPMOD64 && + relocation_type != KZT_X86_64_R_TPOFF64) { + continue; + } + symbol_index = (uint32_t)(relocation.info >> 32); + ownership_proven = symbol_index == 0; + memset(&symbol, 0, sizeof(symbol)); + if (symbol_index && symbol_table_addr) { + uint8_t binding; + uint8_t visibility; + int defined_tls; -static void kzt_public_loader_retain_live_state( - uintptr_t *maps, - size_t *map_count, - const kzt_public_loader_object_t *objects, - size_t object_count) -{ - size_t read_index; - size_t write_index = 0; + if (kzt_public_loader_add_offset( + symbol_table_addr, symbol_index, sizeof(symbol), + &entry_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + if (kzt_public_loader_read(reader, entry_addr, + &symbol, sizeof(symbol)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + binding = symbol.info >> 4; + visibility = symbol.other & 0x3; + defined_tls = + symbol.section_index != 0 && + (symbol.info & 0xf) == 6; + ownership_proven = defined_tls && + (binding == 0 || visibility == 2 || visibility == 3); + validation_required = defined_tls && !ownership_proven && + visibility == 0 && + (binding == 1 || binding == 2); + } + if (!ownership_proven && + (relocation_type != KZT_X86_64_R_TPOFF64 || + !validation_required)) { + continue; + } + if (kzt_public_loader_dynamic_pointer( + object, relocation.offset, &target_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + if (kzt_public_loader_read( + reader, target_addr, &relocated_module_id, + sizeof(relocated_module_id)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (relocation_type == KZT_X86_64_R_TPOFF64) { + int64_t relocated_offset = + (int64_t)relocated_module_id; + __int128 symbol_offset; + __int128 candidate; + + if (!relocated_module_id) { + tls_object->static_tls_offset_pending = 1; + continue; + } - for (read_index = 0; read_index < *map_count; ++read_index) { - if (kzt_public_loader_objects_contain( - objects, object_count, maps[read_index])) { - maps[write_index++] = maps[read_index]; + symbol_offset = + (__int128)symbol.value + relocation.addend; + candidate = relocated_offset - symbol_offset; + if (candidate >= 0 || candidate < INTPTR_MIN || + candidate > INTPTR_MAX) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (tls_object->static_tls_offset_valid && + tls_object->static_tls_offset != (intptr_t)candidate) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (ownership_proven) { + tls_object->static_tls_symbol_value = 0; + tls_object->static_tls_symbol_name_addr = 0; + tls_object->static_tls_offset_needs_validation = 0; + } else if (validation_required && + (!tls_object->static_tls_offset_valid || + tls_object->static_tls_offset_needs_validation)) { + kzt_public_loader_result_t name_result; + + if (!string_table_addr || + symbol.name >= string_table_size || + string_table_addr > UINTPTR_MAX - symbol.name) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + name_result = kzt_public_loader_validate_symbol_name( + reader, string_table_addr + symbol.name, + string_table_size - symbol.name); + if (name_result != KZT_PUBLIC_LOADER_OK) { + return name_result; + } + tls_object->static_tls_symbol_value = symbol.value; + tls_object->static_tls_symbol_name_addr = + string_table_addr + symbol.name; + tls_object->static_tls_offset_needs_validation = 1; + } + tls_object->static_tls_offset = (intptr_t)candidate; + tls_object->static_tls_offset_valid = 1; + continue; + } + if (!relocated_module_id) { + continue; + } + if (relocated_module_id > SIZE_MAX) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (tls_object->module_id && + tls_object->module_id != (size_t)relocated_module_id) { + return KZT_PUBLIC_LOADER_INVALID_STATE; } + tls_object->module_id = (size_t)relocated_module_id; } - *map_count = write_index; + if (tls_object->static_tls_offset_pending) { + return KZT_PUBLIC_LOADER_BUSY; + } + return KZT_PUBLIC_LOADER_OK; } -static int kzt_public_loader_relro_matches( +kzt_public_loader_result_t kzt_public_loader_read_tls_object( + const kzt_public_loader_object_t *object, const kzt_public_loader_reader_t *reader, - uintptr_t load_bias, - uintptr_t protect_start, - uintptr_t protect_end, - size_t page_size) + kzt_public_loader_tls_object_t *tls_object, + int *has_tls) { kzt_x86_64_elf_header_t header; - uintptr_t page_mask = page_size - 1; uintptr_t phdr_base; size_t index; - if (!load_bias) { - return 0; + *has_tls = 0; + if (!object->load_bias) { + return KZT_PUBLIC_LOADER_OK; } - if (kzt_public_loader_read(reader, load_bias, + if (kzt_public_loader_read(reader, object->load_bias, &header, sizeof(header)) != 0) { - return -1; + return KZT_PUBLIC_LOADER_READ_ERROR; } if (header.ident[0] != 0x7f || header.ident[1] != 'E' || header.ident[2] != 'L' || header.ident[3] != 'F' || @@ -499,65 +739,404 @@ static int kzt_public_loader_relro_matches( header.ident[6] != KZT_X86_64_EV_CURRENT || header.phentsize != sizeof(kzt_x86_64_program_header_t) || !header.phnum || header.phnum > KZT_PUBLIC_LOADER_MAX_OBJECTS || - header.phoff > UINTPTR_MAX - load_bias) { - return 0; + header.phoff > UINTPTR_MAX - object->load_bias) { + return KZT_PUBLIC_LOADER_INVALID_STATE; } - phdr_base = load_bias + (uintptr_t)header.phoff; - + phdr_base = object->load_bias + (uintptr_t)header.phoff; for (index = 0; index < header.phnum; ++index) { kzt_x86_64_program_header_t phdr; uintptr_t phdr_addr; - uintptr_t relro_start; - uintptr_t relro_end; + kzt_public_loader_result_t result; if (kzt_public_loader_add_offset( phdr_base, index, sizeof(phdr), &phdr_addr) != 0) { - return -1; + return KZT_PUBLIC_LOADER_OVERFLOW; } if (kzt_public_loader_read(reader, phdr_addr, &phdr, sizeof(phdr)) != 0) { - return -1; + return KZT_PUBLIC_LOADER_READ_ERROR; } - if (phdr.type != KZT_X86_64_PT_GNU_RELRO || !phdr.memsz) { + if (phdr.type != KZT_X86_64_PT_TLS || !phdr.memsz) { continue; } - if (phdr.vaddr > UINTPTR_MAX - load_bias) { - return -1; + if (phdr.filesz > phdr.memsz || phdr.memsz > SIZE_MAX || + phdr.filesz > SIZE_MAX || phdr.align > SIZE_MAX || + phdr.vaddr > UINTPTR_MAX - object->load_bias) { + return KZT_PUBLIC_LOADER_INVALID_STATE; } - relro_start = load_bias + (uintptr_t)phdr.vaddr; - if (phdr.memsz > UINTPTR_MAX - relro_start || - relro_start + (uintptr_t)phdr.memsz > - UINTPTR_MAX - page_mask) { - return -1; + *tls_object = (kzt_public_loader_tls_object_t) { + .link_map_addr = object->link_map_addr, + .load_bias = object->load_bias, + .dynamic_addr = object->dynamic_addr, + .image_addr = object->load_bias + (uintptr_t)phdr.vaddr, + .file_size = (size_t)phdr.filesz, + .memory_size = (size_t)phdr.memsz, + .alignment = phdr.align ? (size_t)phdr.align : 1, + }; + if ((tls_object->alignment & (tls_object->alignment - 1)) != 0) { + return KZT_PUBLIC_LOADER_INVALID_STATE; } - relro_end = (relro_start + (uintptr_t)phdr.memsz + page_mask) & - ~page_mask; - relro_start &= ~page_mask; - /* - * Accept a loader that protects one RELRO range in several calls, - * or one protection call that fully contains this RELRO range. A - * mere overlap is ambiguous and must stay on the guest path. - */ - return (protect_start >= relro_start && protect_end <= relro_end) || - (relro_start >= protect_start && relro_end <= protect_end); + tls_object->first_byte_offset = + (size_t)phdr.vaddr & (tls_object->alignment - 1); + result = kzt_public_loader_tls_relocations( + object, reader, tls_object); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + *has_tls = 1; + return KZT_PUBLIC_LOADER_OK; } - return 0; + return KZT_PUBLIC_LOADER_OK; } -kzt_public_loader_result_t kzt_public_loader_object_has_relro( - const kzt_public_loader_object_t *object, +kzt_public_loader_result_t kzt_public_loader_materialize_tls_image( + const kzt_public_loader_tls_object_t *object, const kzt_public_loader_reader_t *reader, - int *has_relro) + void *destination, + size_t destination_size) { - kzt_x86_64_elf_header_t header; - uintptr_t phdr_base; - size_t index; + kzt_public_loader_object_t loader_object = { + .load_bias = object ? object->load_bias : 0, + }; + uint64_t rela_value = 0; + uint64_t rela_size = 0; + uint64_t rela_entry_size = 0; + uint64_t symbol_table_value = 0; + uint64_t symbol_entry_size = 0; + uintptr_t rela_addr = 0; + uintptr_t symbol_table_addr = 0; + int terminated = 0; - if (!object || !object->load_bias || !reader || - !reader->read_memory || !has_relro) { + if (!object || !reader || !reader->read_memory || !destination || + object->file_size > destination_size || + (object->file_size && !object->image_addr)) { return KZT_PUBLIC_LOADER_INVALID_INPUT; } - *has_relro = 0; + if (object->file_size && + kzt_public_loader_read( + reader, object->image_addr, + destination, object->file_size) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (!object->file_size || !object->dynamic_addr) { + return KZT_PUBLIC_LOADER_OK; + } + for (size_t index = 0; + index < KZT_PUBLIC_LOADER_MAX_DYNAMIC_ENTRIES; ++index) { + kzt_x86_64_dynamic_entry_t entry; + uintptr_t entry_addr; + + if (kzt_public_loader_add_offset( + object->dynamic_addr, index, sizeof(entry), + &entry_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + if (kzt_public_loader_read( + reader, entry_addr, &entry, sizeof(entry)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (entry.tag == KZT_X86_64_DT_NULL) { + terminated = 1; + break; + } + switch (entry.tag) { + case KZT_X86_64_DT_RELA: + rela_value = entry.value; + break; + case KZT_X86_64_DT_RELASZ: + rela_size = entry.value; + break; + case KZT_X86_64_DT_RELAENT: + rela_entry_size = entry.value; + break; + case KZT_X86_64_DT_SYMTAB: + symbol_table_value = entry.value; + break; + case KZT_X86_64_DT_SYMENT: + symbol_entry_size = entry.value; + break; + default: + break; + } + } + if (!terminated) { + return KZT_PUBLIC_LOADER_LIMIT; + } + if (!rela_value || !rela_size) { + return KZT_PUBLIC_LOADER_OK; + } + if (rela_entry_size != sizeof(kzt_x86_64_relocation_t) || + rela_size % rela_entry_size || + rela_size / rela_entry_size > KZT_PUBLIC_LOADER_MAX_SYMBOLS || + kzt_public_loader_dynamic_pointer( + &loader_object, + rela_value, &rela_addr) != 0) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (symbol_table_value && + (symbol_entry_size != sizeof(kzt_x86_64_symbol_t) || + kzt_public_loader_dynamic_pointer( + &loader_object, + symbol_table_value, &symbol_table_addr) != 0)) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + for (size_t index = 0; + index < rela_size / rela_entry_size; ++index) { + kzt_x86_64_relocation_t relocation; + uintptr_t relocation_addr; + uintptr_t target_addr; + uintptr_t image_end; + uint32_t relocation_type; + __int128 value; + int value_from_loader = 0; + + if (kzt_public_loader_add_offset( + rela_addr, index, sizeof(relocation), + &relocation_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + if (kzt_public_loader_read( + reader, relocation_addr, + &relocation, sizeof(relocation)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (kzt_public_loader_dynamic_pointer( + &loader_object, + relocation.offset, &target_addr) != 0 || + object->image_addr > UINTPTR_MAX - object->file_size) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + image_end = object->image_addr + object->file_size; + if (target_addr < object->image_addr || target_addr >= image_end) { + continue; + } + if (target_addr > image_end - sizeof(uint64_t)) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + relocation_type = (uint32_t)relocation.info; + value = (__int128)object->load_bias + relocation.addend; + if (relocation_type == KZT_X86_64_R_64) { + kzt_x86_64_symbol_t symbol; + uintptr_t symbol_addr; + uint32_t symbol_index = + (uint32_t)(relocation.info >> 32); + uint8_t binding; + uint8_t visibility; + uint8_t symbol_type; + + if (!symbol_index || !symbol_table_addr) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (kzt_public_loader_add_offset( + symbol_table_addr, symbol_index, sizeof(symbol), + &symbol_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + if (kzt_public_loader_read( + reader, symbol_addr, + &symbol, sizeof(symbol)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + binding = symbol.info >> 4; + visibility = symbol.other & 0x3; + symbol_type = symbol.info & 0xf; + if (symbol_type != 10 && symbol.section_index && + (binding == 0 || visibility == 2 || visibility == 3)) { + value += symbol.value; + } else { + uint64_t relocated_value; + + if (kzt_public_loader_read( + reader, target_addr, &relocated_value, + sizeof(relocated_value)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + value = relocated_value; + value_from_loader = 1; + } + } else if (relocation_type != KZT_X86_64_R_RELATIVE) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (!value_from_loader && + (value < 0 || value > UINTPTR_MAX)) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + memcpy((unsigned char *)destination + + (target_addr - object->image_addr), + &(uint64_t){ (uint64_t)value }, sizeof(uint64_t)); + } + return KZT_PUBLIC_LOADER_OK; +} + +static kzt_public_loader_result_t kzt_public_loader_find_r_debug( + uintptr_t dynamic_addr, + size_t max_dynamic_entries, + const kzt_public_loader_reader_t *reader, + uintptr_t *r_debug_addr) +{ + size_t index; + + if (!dynamic_addr || !max_dynamic_entries || !reader || + !reader->read_memory || !r_debug_addr) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + *r_debug_addr = 0; + + for (index = 0; index < max_dynamic_entries; ++index) { + kzt_x86_64_dynamic_entry_t entry; + uintptr_t entry_addr; + + if (kzt_public_loader_add_offset( + dynamic_addr, index, sizeof(entry), &entry_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + if (kzt_public_loader_read( + reader, entry_addr, &entry, sizeof(entry)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (entry.tag == KZT_X86_64_DT_DEBUG && entry.value) { + *r_debug_addr = (uintptr_t)entry.value; + return KZT_PUBLIC_LOADER_OK; + } + if (entry.tag == KZT_X86_64_DT_NULL) { + return KZT_PUBLIC_LOADER_NOT_FOUND; + } + } + return KZT_PUBLIC_LOADER_NOT_FOUND; +} + +static int kzt_public_loader_contains(const uintptr_t *maps, + size_t map_count, + uintptr_t map_addr) +{ + size_t index; + + for (index = 0; index < map_count; ++index) { + if (maps[index] == map_addr) { + return 1; + } + } + return 0; +} + +static int kzt_public_loader_objects_contain( + const kzt_public_loader_object_t *objects, + size_t object_count, + uintptr_t map_addr) +{ + size_t index; + + for (index = 0; index < object_count; ++index) { + if (objects[index].link_map_addr == map_addr) { + return 1; + } + } + return 0; +} + +static void kzt_public_loader_retain_live_state( + uintptr_t *maps, + size_t *map_count, + const kzt_public_loader_object_t *objects, + size_t object_count) +{ + size_t read_index; + size_t write_index = 0; + + for (read_index = 0; read_index < *map_count; ++read_index) { + if (kzt_public_loader_objects_contain( + objects, object_count, maps[read_index])) { + maps[write_index++] = maps[read_index]; + } + } + *map_count = write_index; +} + +static int kzt_public_loader_relro_matches( + const kzt_public_loader_reader_t *reader, + uintptr_t load_bias, + uintptr_t protect_start, + uintptr_t protect_end, + size_t page_size) +{ + kzt_x86_64_elf_header_t header; + uintptr_t page_mask = page_size - 1; + uintptr_t phdr_base; + size_t index; + + if (!load_bias) { + return 0; + } + if (kzt_public_loader_read(reader, load_bias, + &header, sizeof(header)) != 0) { + return -1; + } + if (header.ident[0] != 0x7f || header.ident[1] != 'E' || + header.ident[2] != 'L' || header.ident[3] != 'F' || + header.ident[4] != KZT_X86_64_ELFCLASS64 || + header.ident[5] != KZT_X86_64_ELFDATA2LSB || + header.ident[6] != KZT_X86_64_EV_CURRENT || + header.phentsize != sizeof(kzt_x86_64_program_header_t) || + !header.phnum || header.phnum > KZT_PUBLIC_LOADER_MAX_OBJECTS || + header.phoff > UINTPTR_MAX - load_bias) { + return 0; + } + phdr_base = load_bias + (uintptr_t)header.phoff; + + for (index = 0; index < header.phnum; ++index) { + kzt_x86_64_program_header_t phdr; + uintptr_t phdr_addr; + uintptr_t relro_start; + uintptr_t relro_end; + + if (kzt_public_loader_add_offset( + phdr_base, index, sizeof(phdr), &phdr_addr) != 0) { + return -1; + } + if (kzt_public_loader_read(reader, phdr_addr, + &phdr, sizeof(phdr)) != 0) { + return -1; + } + if (phdr.type != KZT_X86_64_PT_GNU_RELRO || !phdr.memsz) { + continue; + } + if (phdr.vaddr > UINTPTR_MAX - load_bias) { + return -1; + } + relro_start = load_bias + (uintptr_t)phdr.vaddr; + if (phdr.memsz > UINTPTR_MAX - relro_start || + relro_start + (uintptr_t)phdr.memsz > + UINTPTR_MAX - page_mask) { + return -1; + } + relro_end = (relro_start + (uintptr_t)phdr.memsz + page_mask) & + ~page_mask; + relro_start &= ~page_mask; + /* + * Accept a loader that protects one RELRO range in several calls, + * or one protection call that fully contains this RELRO range. A + * mere overlap is ambiguous and must stay on the guest path. + */ + return (protect_start >= relro_start && protect_end <= relro_end) || + (relro_start >= protect_start && relro_end <= protect_end); + } + return 0; +} + +kzt_public_loader_result_t kzt_public_loader_object_has_relro( + const kzt_public_loader_object_t *object, + const kzt_public_loader_reader_t *reader, + int *has_relro) +{ + kzt_x86_64_elf_header_t header; + uintptr_t phdr_base; + size_t index; + + if (!object || !object->load_bias || !reader || + !reader->read_memory || !has_relro) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + *has_relro = 0; if (kzt_public_loader_read(reader, object->load_bias, &header, sizeof(header)) != 0) { return KZT_PUBLIC_LOADER_READ_ERROR; @@ -604,6 +1183,7 @@ static kzt_public_loader_result_t kzt_public_loader_capture( { kzt_x86_64_r_debug_t debug; kzt_public_loader_object_t objects[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + uint64_t object_generations[KZT_PUBLIC_LOADER_MAX_OBJECTS]; uintptr_t current; size_t count = 0; size_t index; @@ -657,6 +1237,26 @@ static kzt_public_loader_result_t kzt_public_loader_capture( } for (index = 0; index < count; ++index) { + size_t previous_index; + + object_generations[index] = 0; + for (previous_index = 0; + previous_index < observer->live_map_count; + ++previous_index) { + if (observer->live_maps[previous_index] == + objects[index].link_map_addr) { + object_generations[index] = + observer->live_map_generations[previous_index]; + break; + } + } + if (!object_generations[index]) { + if (observer->next_load_generation == UINT64_MAX) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + object_generations[index] = + ++observer->next_load_generation; + } if (!kzt_public_loader_contains(observer->live_maps, observer->live_map_count, objects[index].link_map_addr) && @@ -673,6 +1273,8 @@ static kzt_public_loader_result_t kzt_public_loader_capture( &observer->fallback_reported_map_count, objects, count); for (index = 0; index < count; ++index) { observer->live_maps[index] = objects[index].link_map_addr; + observer->live_map_generations[index] = + object_generations[index]; } observer->live_map_count = count; if (observed_brk) { @@ -926,15 +1528,45 @@ kzt_public_loader_result_t kzt_public_loader_observer_refresh( return result; } -kzt_public_loader_result_t kzt_public_loader_find_symbol( +kzt_public_loader_result_t kzt_public_loader_state_is_consistent( const kzt_public_loader_observer_t *observer, - const kzt_public_loader_reader_t *reader, - const char *symbol_name, - uintptr_t *symbol_addr) + const kzt_public_loader_reader_t *reader) { - uintptr_t found_addr = 0; - size_t symbol_name_size; - size_t index; + kzt_x86_64_r_debug_t debug; + + if (!observer || !observer->active || !observer->r_debug_addr || + !reader || !reader->read_memory) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + if (kzt_public_loader_read(reader, observer->r_debug_addr, + &debug, sizeof(debug)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (debug.version < 1 || !debug.map || !debug.brk || + (observer->r_brk_addr && + observer->r_brk_addr != (uintptr_t)debug.brk)) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (debug.state == KZT_LOADER_DEBUG_ADD || + debug.state == KZT_LOADER_DEBUG_DELETE) { + return KZT_PUBLIC_LOADER_BUSY; + } + if (debug.state != KZT_LOADER_DEBUG_CONSISTENT) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + return KZT_PUBLIC_LOADER_OK; +} + +kzt_public_loader_result_t kzt_public_loader_find_symbol( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader, + const char *symbol_name, + uintptr_t *symbol_addr) +{ + uintptr_t found_addr = 0; + size_t symbol_name_size; + size_t index; + int found = 0; if (!observer || !observer->active || !reader || !reader->read_memory || !symbol_name || !symbol_addr) { @@ -973,18 +1605,598 @@ kzt_public_loader_result_t kzt_public_loader_find_symbol( if (result != KZT_PUBLIC_LOADER_OK) { return result; } - if (found_addr && found_addr != candidate) { + if (found && found_addr != candidate) { return KZT_PUBLIC_LOADER_INVALID_STATE; } found_addr = candidate; + found = 1; + } + if (!found) { + return KZT_PUBLIC_LOADER_NOT_FOUND; + } + *symbol_addr = found_addr; + return KZT_PUBLIC_LOADER_OK; +} + +kzt_public_loader_result_t kzt_public_loader_object_contains_address( + const kzt_public_loader_object_t *object, + const kzt_public_loader_reader_t *reader, + uintptr_t guest_addr, + int *contains) +{ + kzt_x86_64_elf_header_t header; + uintptr_t phdr_base; + uintptr_t phdr_last; + uintptr_t load_bias; + size_t index; + + if (!object || !reader || !reader->read_memory || + !guest_addr || !contains) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + *contains = 0; + load_bias = object->load_bias; + + if (!load_bias) { + return KZT_PUBLIC_LOADER_OK; + } + if (kzt_public_loader_read(reader, load_bias, + &header, sizeof(header)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (header.ident[0] != 0x7f || header.ident[1] != 'E' || + header.ident[2] != 'L' || header.ident[3] != 'F' || + header.ident[4] != KZT_X86_64_ELFCLASS64 || + header.ident[5] != KZT_X86_64_ELFDATA2LSB || + header.ident[6] != KZT_X86_64_EV_CURRENT || + (header.type != KZT_X86_64_ET_EXEC && + header.type != KZT_X86_64_ET_DYN) || + header.machine != KZT_X86_64_EM_X86_64 || + header.version != KZT_X86_64_EV_CURRENT || + header.ehsize != sizeof(header) || + !header.phoff || + header.phentsize != sizeof(kzt_x86_64_program_header_t) || + !header.phnum) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (header.phnum > KZT_PUBLIC_LOADER_MAX_OBJECTS) { + return KZT_PUBLIC_LOADER_LIMIT; + } + if (header.phoff > UINTPTR_MAX - load_bias) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + phdr_base = load_bias + (uintptr_t)header.phoff; + if (kzt_public_loader_add_offset( + phdr_base, header.phnum - 1, sizeof(kzt_x86_64_program_header_t), + &phdr_last) != 0 || + phdr_last > UINTPTR_MAX - sizeof(kzt_x86_64_program_header_t)) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + + for (index = 0; index < header.phnum; ++index) { + kzt_x86_64_program_header_t phdr; + uintptr_t phdr_addr; + uintptr_t segment_start; + uintptr_t segment_end; + + if (kzt_public_loader_add_offset( + phdr_base, index, sizeof(phdr), &phdr_addr) != 0) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + if (kzt_public_loader_read(reader, phdr_addr, + &phdr, sizeof(phdr)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (phdr.type != KZT_X86_64_PT_LOAD) { + continue; + } + if (phdr.filesz > phdr.memsz || + (phdr.align > 1 && + ((phdr.align & (phdr.align - 1)) != 0 || + phdr.vaddr % phdr.align != phdr.offset % phdr.align))) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (phdr.filesz > UINT64_MAX - phdr.offset || + phdr.memsz > UINT64_MAX - phdr.vaddr || + phdr.vaddr > UINTPTR_MAX - load_bias) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + segment_start = load_bias + (uintptr_t)phdr.vaddr; + if (phdr.memsz > UINTPTR_MAX - segment_start) { + return KZT_PUBLIC_LOADER_OVERFLOW; + } + segment_end = segment_start + (uintptr_t)phdr.memsz; + if (phdr.memsz && guest_addr >= segment_start && + guest_addr < segment_end) { + *contains = 1; + } + } + return KZT_PUBLIC_LOADER_OK; +} + +kzt_public_loader_result_t kzt_public_loader_find_object_by_address( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader, + uintptr_t guest_addr, + kzt_public_loader_object_t *object) +{ + kzt_public_loader_object_t candidate = { 0 }; + kzt_x86_64_r_debug_t debug; + kzt_x86_64_r_debug_t final_debug; + uintptr_t current_addr; + uintptr_t previous_addr = 0; + size_t count = 0; + int found = 0; + + if (!observer || !observer->active || !observer->r_debug_addr || + !reader || !reader->read_memory || !guest_addr || !object) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + memset(object, 0, sizeof(*object)); + if (kzt_public_loader_read(reader, observer->r_debug_addr, + &debug, sizeof(debug)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (debug.version < 1 || + debug.state < KZT_LOADER_DEBUG_CONSISTENT || + debug.state > KZT_LOADER_DEBUG_DELETE || !debug.map || !debug.brk || + (uintptr_t)debug.brk != observer->r_brk_addr) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (debug.state == KZT_LOADER_DEBUG_DELETE) { + return KZT_PUBLIC_LOADER_BUSY; + } + current_addr = (uintptr_t)debug.map; + while (current_addr) { + kzt_x86_64_link_map_prefix_t map; + kzt_public_loader_object_t current; + kzt_public_loader_result_t result; + int contains; + + if (++count > UINT16_MAX) { + return KZT_PUBLIC_LOADER_LIMIT; + } + if (kzt_public_loader_read(reader, current_addr, + &map, sizeof(map)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if ((uintptr_t)map.previous != previous_addr) { + return debug.state == KZT_LOADER_DEBUG_ADD + ? KZT_PUBLIC_LOADER_BUSY + : KZT_PUBLIC_LOADER_INVALID_STATE; + } + current = (kzt_public_loader_object_t) { + .link_map_addr = current_addr, + .load_bias = (uintptr_t)map.load_bias, + .name_addr = (uintptr_t)map.name, + .dynamic_addr = (uintptr_t)map.dynamic_addr, + .next_addr = (uintptr_t)map.next, + .previous_addr = (uintptr_t)map.previous, + }; + result = kzt_public_loader_object_contains_address( + ¤t, reader, guest_addr, &contains); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + previous_addr = current_addr; + current_addr = (uintptr_t)map.next; + if (!contains) { + continue; + } + if (found) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + candidate = current; + found = 1; + } + if (kzt_public_loader_read(reader, observer->r_debug_addr, + &final_debug, sizeof(final_debug)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (final_debug.state != debug.state) { + return KZT_PUBLIC_LOADER_BUSY; + } + if (final_debug.version != debug.version || + final_debug.map != debug.map || final_debug.brk != debug.brk) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (!found) { + if (debug.state == KZT_LOADER_DEBUG_ADD) { + return KZT_PUBLIC_LOADER_BUSY; + } + return KZT_PUBLIC_LOADER_NOT_FOUND; + } + *object = candidate; + return KZT_PUBLIC_LOADER_OK; +} + +static kzt_public_loader_result_t kzt_public_loader_collect_tls_filtered( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader, + uintptr_t link_map_filter, + kzt_public_loader_tls_object_t *objects, + size_t object_capacity, + size_t *object_count) +{ + size_t index; + + if (!observer || !observer->active || !reader || + !reader->read_memory || !objects || !object_capacity || + !object_count) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + *object_count = 0; + for (index = 0; index < observer->live_map_count; ++index) { + kzt_x86_64_link_map_prefix_t map; + kzt_public_loader_object_t object; + kzt_public_loader_tls_object_t tls_object; + kzt_public_loader_result_t result; + int has_tls; + + if (link_map_filter && + observer->live_maps[index] != link_map_filter) { + continue; + } + if (kzt_public_loader_read(reader, observer->live_maps[index], + &map, sizeof(map)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + object = (kzt_public_loader_object_t) { + .link_map_addr = observer->live_maps[index], + .load_bias = (uintptr_t)map.load_bias, + .name_addr = (uintptr_t)map.name, + .dynamic_addr = (uintptr_t)map.dynamic_addr, + .next_addr = (uintptr_t)map.next, + .previous_addr = (uintptr_t)map.previous, + }; + result = kzt_public_loader_read_tls_object( + &object, reader, &tls_object, &has_tls); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + if (!has_tls) { + continue; + } + if (tls_object.static_tls_offset_needs_validation) { + char symbol_name[KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME + 1]; + uintptr_t resolved_addr; + uintptr_t expected_addr; + + result = kzt_public_loader_copy_symbol_name( + reader, tls_object.static_tls_symbol_name_addr, + symbol_name); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + result = kzt_public_loader_find_symbol( + observer, reader, symbol_name, &resolved_addr); + if (result != KZT_PUBLIC_LOADER_OK || + tls_object.static_tls_symbol_value > + UINTPTR_MAX - tls_object.load_bias) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + expected_addr = tls_object.load_bias + + tls_object.static_tls_symbol_value; + if (resolved_addr != expected_addr) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + tls_object.static_tls_symbol_value = 0; + tls_object.static_tls_symbol_name_addr = 0; + tls_object.static_tls_offset_needs_validation = 0; + } + if (*object_count == object_capacity) { + return KZT_PUBLIC_LOADER_LIMIT; + } + tls_object.load_generation = + observer->live_map_generations[index]; + objects[(*object_count)++] = tls_object; + } + return KZT_PUBLIC_LOADER_OK; +} + +static kzt_public_loader_result_t kzt_public_loader_find_symbol_live( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader, + const char *symbol_name, + uintptr_t *symbol_addr) +{ + kzt_x86_64_r_debug_t debug; + kzt_x86_64_r_debug_t final_debug; + uintptr_t current; + uintptr_t previous = 0; + uintptr_t found_addr = 0; + size_t symbol_name_size; + size_t count = 0; + int found = 0; + + if (!observer || !observer->active || !observer->r_debug_addr || + !reader || !reader->read_memory || !symbol_name || !symbol_addr) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + symbol_name_size = strlen(symbol_name); + if (!symbol_name_size || + symbol_name_size > KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + *symbol_addr = 0; + if (kzt_public_loader_read(reader, observer->r_debug_addr, + &debug, sizeof(debug)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (debug.version < 1 || + (debug.state != KZT_LOADER_DEBUG_CONSISTENT && + debug.state != KZT_LOADER_DEBUG_ADD) || !debug.map || !debug.brk || + (observer->r_brk_addr && + observer->r_brk_addr != (uintptr_t)debug.brk)) { + return debug.state == KZT_LOADER_DEBUG_ADD || + debug.state == KZT_LOADER_DEBUG_DELETE + ? KZT_PUBLIC_LOADER_BUSY + : KZT_PUBLIC_LOADER_INVALID_STATE; + } + + current = (uintptr_t)debug.map; + while (current) { + kzt_x86_64_link_map_prefix_t map; + kzt_public_loader_object_t object; + kzt_public_loader_result_t result; + uintptr_t candidate = 0; + + if (++count > UINT16_MAX) { + return KZT_PUBLIC_LOADER_LIMIT; + } + if (kzt_public_loader_read(reader, current, + &map, sizeof(map)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if ((uintptr_t)map.previous != previous) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + object = (kzt_public_loader_object_t) { + .link_map_addr = current, + .load_bias = (uintptr_t)map.load_bias, + .name_addr = (uintptr_t)map.name, + .dynamic_addr = (uintptr_t)map.dynamic_addr, + .next_addr = (uintptr_t)map.next, + .previous_addr = (uintptr_t)map.previous, + }; + result = kzt_public_loader_find_object_symbol( + &object, reader, symbol_name, symbol_name_size, &candidate); + if (result != KZT_PUBLIC_LOADER_OK && + result != KZT_PUBLIC_LOADER_NOT_FOUND) { + return result; + } + if (result == KZT_PUBLIC_LOADER_OK) { + if (found && found_addr != candidate) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + found_addr = candidate; + found = 1; + } + previous = current; + current = (uintptr_t)map.next; + } + if (kzt_public_loader_read(reader, observer->r_debug_addr, + &final_debug, sizeof(final_debug)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; } - if (!found_addr) { + if (final_debug.version != debug.version || + final_debug.map != debug.map || final_debug.brk != debug.brk || + final_debug.state != debug.state) { + return KZT_PUBLIC_LOADER_BUSY; + } + if (!found) { return KZT_PUBLIC_LOADER_NOT_FOUND; } *symbol_addr = found_addr; return KZT_PUBLIC_LOADER_OK; } +static kzt_public_loader_result_t kzt_public_loader_collect_tls_live( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader, + uintptr_t link_map_filter, + kzt_public_loader_tls_object_t *objects, + size_t object_capacity, + size_t *object_count) +{ + kzt_x86_64_r_debug_t debug; + kzt_x86_64_r_debug_t final_debug; + uintptr_t current; + uintptr_t previous = 0; + size_t count = 0; + + if (!observer || !observer->active || !observer->r_debug_addr || + !reader || !reader->read_memory || !objects || !object_capacity || + !object_count) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + *object_count = 0; + if (kzt_public_loader_read(reader, observer->r_debug_addr, + &debug, sizeof(debug)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (debug.version < 1 || + (debug.state != KZT_LOADER_DEBUG_CONSISTENT && + debug.state != KZT_LOADER_DEBUG_ADD) || !debug.map || !debug.brk || + (observer->r_brk_addr && + observer->r_brk_addr != (uintptr_t)debug.brk)) { + return debug.state == KZT_LOADER_DEBUG_ADD || + debug.state == KZT_LOADER_DEBUG_DELETE + ? KZT_PUBLIC_LOADER_BUSY + : KZT_PUBLIC_LOADER_INVALID_STATE; + } + + current = (uintptr_t)debug.map; + while (current) { + kzt_x86_64_link_map_prefix_t map; + kzt_public_loader_object_t object; + kzt_public_loader_tls_object_t tls_object; + kzt_public_loader_result_t result; + uint64_t load_generation = 0; + int has_tls; + + if (++count > UINT16_MAX) { + return KZT_PUBLIC_LOADER_LIMIT; + } + if (kzt_public_loader_read(reader, current, + &map, sizeof(map)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if ((uintptr_t)map.previous != previous) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (!link_map_filter || current == link_map_filter) { + object = (kzt_public_loader_object_t) { + .link_map_addr = current, + .load_bias = (uintptr_t)map.load_bias, + .name_addr = (uintptr_t)map.name, + .dynamic_addr = (uintptr_t)map.dynamic_addr, + .next_addr = (uintptr_t)map.next, + .previous_addr = (uintptr_t)map.previous, + }; + result = kzt_public_loader_read_tls_object( + &object, reader, &tls_object, &has_tls); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + if (has_tls) { + if (tls_object.static_tls_offset_needs_validation) { + char symbol_name[KZT_PUBLIC_LOADER_MAX_SYMBOL_NAME + 1]; + uintptr_t resolved_addr; + uintptr_t expected_addr; + + result = kzt_public_loader_copy_symbol_name( + reader, tls_object.static_tls_symbol_name_addr, + symbol_name); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + result = kzt_public_loader_find_symbol_live( + observer, reader, symbol_name, &resolved_addr); + if (result != KZT_PUBLIC_LOADER_OK || + tls_object.static_tls_symbol_value > + UINTPTR_MAX - tls_object.load_bias) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + expected_addr = tls_object.load_bias + + tls_object.static_tls_symbol_value; + if (resolved_addr != expected_addr) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + tls_object.static_tls_symbol_value = 0; + tls_object.static_tls_symbol_name_addr = 0; + tls_object.static_tls_offset_needs_validation = 0; + } + if (*object_count == object_capacity) { + return KZT_PUBLIC_LOADER_LIMIT; + } + for (size_t index = 0; + index < observer->live_map_count; ++index) { + if (observer->live_maps[index] == current) { + load_generation = + observer->live_map_generations[index]; + break; + } + } + tls_object.load_generation = + load_generation ? load_generation : (uint64_t)current; + objects[(*object_count)++] = tls_object; + } + } + previous = current; + current = (uintptr_t)map.next; + } + if (link_map_filter && !*object_count) { + return KZT_PUBLIC_LOADER_NOT_FOUND; + } + if (kzt_public_loader_read(reader, observer->r_debug_addr, + &final_debug, sizeof(final_debug)) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (final_debug.state != debug.state) { + return KZT_PUBLIC_LOADER_BUSY; + } + if (final_debug.version != debug.version || + final_debug.map != debug.map || final_debug.brk != debug.brk) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + return KZT_PUBLIC_LOADER_OK; +} + +kzt_public_loader_result_t kzt_public_loader_collect_tls( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader, + kzt_public_loader_tls_object_t *objects, + size_t object_capacity, + size_t *object_count) +{ + return kzt_public_loader_collect_tls_filtered( + observer, reader, 0, objects, object_capacity, object_count); +} + +static int kzt_public_loader_snapshot_visit( + const kzt_public_loader_object_t *object, + void *opaque) +{ + (void)object; + (void)opaque; + return 0; +} + +kzt_public_loader_result_t kzt_public_loader_snapshot_tls( + const kzt_public_loader_observer_t *observer, + uintptr_t dynamic_addr, + size_t max_dynamic_entries, + const kzt_public_loader_reader_t *reader, + uintptr_t link_map_filter, + kzt_public_loader_tls_object_t *objects, + size_t object_capacity, + size_t *object_count) +{ + kzt_public_loader_observer_t snapshot; + kzt_public_loader_result_t result; + + if (!observer || !reader || !reader->read_memory || !objects || + !object_capacity || !object_count) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + snapshot = *observer; + if (snapshot.active) { + result = kzt_public_loader_observer_refresh( + &snapshot, reader, kzt_public_loader_snapshot_visit, NULL); + if (result == KZT_PUBLIC_LOADER_BUSY) { + return kzt_public_loader_collect_tls_live( + &snapshot, reader, link_map_filter, + objects, object_capacity, object_count); + } + } else { + result = kzt_public_loader_observer_activate( + &snapshot, dynamic_addr, max_dynamic_entries, reader, + kzt_public_loader_snapshot_visit, NULL); + } + if (result != KZT_PUBLIC_LOADER_OK) { + if (result == KZT_PUBLIC_LOADER_LIMIT) { + if (!snapshot.active) { + result = kzt_public_loader_find_r_debug( + dynamic_addr, max_dynamic_entries, reader, + &snapshot.r_debug_addr); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + snapshot.active = 1; + } + result = kzt_public_loader_collect_tls_live( + &snapshot, reader, link_map_filter, + objects, object_capacity, object_count); + return result; + } + return result; + } + return kzt_public_loader_collect_tls_filtered( + &snapshot, reader, link_map_filter, + objects, object_capacity, object_count); +} + const char *kzt_public_loader_result_name( kzt_public_loader_result_t result) { diff --git a/target/i386/latx/context/meson.build b/target/i386/latx/context/meson.build index 27df375816..ad850e98c5 100644 --- a/target/i386/latx/context/meson.build +++ b/target/i386/latx/context/meson.build @@ -95,4 +95,7 @@ my_file = files( 'wrappertbbridge.c' ) + i386_ss.add(when: 'CONFIG_LATX', if_true: my_file) +i386_ss.add(when: 'CONFIG_LATX_KZT', if_true: files('kzt-guest-tls.c')) +i386_ss.add(when: 'CONFIG_LATX_KZT', if_true: files('kzt-guest-thread.c')) diff --git a/target/i386/latx/context/myalign.c b/target/i386/latx/context/myalign.c index 3950b17d29..a128269dd3 100644 --- a/target/i386/latx/context/myalign.c +++ b/target/i386/latx/context/myalign.c @@ -8,20 +8,25 @@ #include "config-host.h" #include "qemu/path.h" + +#include "callback.h" #include "lsenv.h" #include "myalign.h" #include "elfloader.h" #include "elfloader_private.h" #include "kzt-groups.h" #include "kzt_public_loader_observer.h" +#include "kzt-guest-tls.h" #include "kzt_relro_preprotect.h" #include "latx-options.h" #include "librarian_private.h" #include "library_private.h" +#include #include #include #include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmissing-prototypes" @@ -32,6 +37,8 @@ int box64_x87_no80bits = 1; static TranslationBlock *kzt_install_guest_pc_callback( uint32_t *inst_old, CPUState *cpu, uintptr_t addr, void (*callback)(CPUX86State *)); +static void kzt_guest_fork_entry_callback(CPUX86State *env); +static void kzt_install_guest_fork_callback(CPUState *cpu); #ifndef HUGE_VAL #define HUGE_VAL (1.0 / 0.0) @@ -2094,6 +2101,7 @@ static void init_main_elf(elfheader_t* elf_header,int fd, uintptr_t load_addr, ResetSpecialCaseMainElf(elf_header); } int wine_option_kzt; + int kzt_init(char** argv, int argc,char** target_argv, int target_argc, struct linux_binprm* bprm) { if (!option_kzt && !wine_option_kzt) { @@ -2128,10 +2136,22 @@ int kzt_init(char** argv, int argc,char** target_argv, int target_argc, } static kzt_public_loader_observer_t kzt_public_loader_observer; +typedef struct kzt_guest_tls_external_map { + uintptr_t link_map_addr; + uint64_t load_generation; + size_t references; +} kzt_guest_tls_external_map_t; +static kzt_guest_tls_external_map_t + kzt_guest_tls_external_maps[KZT_PUBLIC_LOADER_MAX_OBJECTS]; +static size_t kzt_guest_tls_external_map_count; +static uint64_t kzt_guest_tls_external_generation; +static int kzt_guest_tls_external_error; static int kzt_main_relocated_before_relro; static int kzt_main_fallback_reported; static int kzt_observer_failure_reported; static uint32 kzt_public_r_brk_inst[2]; +static uint32 kzt_guest_fork_inst[2]; +static uintptr_t kzt_guest_fork_addr; extern void* x86free; extern void* x86realloc; extern void* x86pthread_setcanceltype; @@ -2265,6 +2285,713 @@ static const kzt_public_loader_reader_t kzt_public_loader_reader = { .opaque = NULL, }; +int kzt_guest_loader_state_is_consistent(void) +{ + kzt_public_loader_result_t result; + + mmap_lock(); + result = kzt_public_loader_state_is_consistent( + &kzt_public_loader_observer, &kzt_public_loader_reader); + mmap_unlock(); + return result == KZT_PUBLIC_LOADER_OK; +} + +static int kzt_find_main_dynamic_table(const elfheader_t *head, + uintptr_t *dynamic_addr, + size_t *dynamic_count); + +static int kzt_loader_snapshot_visit( + const kzt_public_loader_object_t *object, void *opaque); + +static kzt_public_loader_result_t kzt_read_guest_loader_object( + uintptr_t link_map_addr, + kzt_public_loader_object_t *object) +{ + kzt_x86_64_link_map_prefix_t map; + + if (!link_map_addr || !object) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + if (kzt_public_loader_read_guest( + link_map_addr, &map, sizeof(map), NULL) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + *object = (kzt_public_loader_object_t) { + .link_map_addr = link_map_addr, + .load_bias = (uintptr_t)map.load_bias, + .name_addr = (uintptr_t)map.name, + .dynamic_addr = (uintptr_t)map.dynamic_addr, + .next_addr = (uintptr_t)map.next, + .previous_addr = (uintptr_t)map.previous, + }; + return KZT_PUBLIC_LOADER_OK; +} + +static kzt_public_loader_result_t kzt_read_guest_object_name( + uintptr_t name_addr, char *name, size_t capacity) +{ + if (!name_addr || !name || capacity < 2) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + for (size_t index = 0; index < capacity; ++index) { + if (name_addr > UINTPTR_MAX - index || + kzt_public_loader_read_guest( + name_addr + index, &name[index], 1, NULL) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (!name[index]) { + return KZT_PUBLIC_LOADER_OK; + } + } + name[capacity - 1] = '\0'; + return KZT_PUBLIC_LOADER_LIMIT; +} + +static int kzt_guest_object_name_matches(const char *candidate, + const char *requested) +{ + const char *base; + const char *suffix; + size_t stem_length; + + if (!candidate || !requested) { + return 0; + } + base = strrchr(candidate, '/'); + base = base ? base + 1 : candidate; + if (strcmp(base, requested) == 0) { + return 1; + } + suffix = strstr(requested, ".so"); + if (!suffix) { + return 0; + } + stem_length = (size_t)(suffix - requested); + return strncmp(base, requested, stem_length) == 0 && + base[stem_length] == '-'; +} + +static kzt_public_loader_result_t +kzt_resolve_guest_object_symbol_live( + const kzt_public_loader_observer_t *observer, + const char *object_name, + const char *symbol_name, + uintptr_t *address) +{ + enum { KZT_GUEST_OBJECT_NAME_MAX = 512 }; + kzt_x86_64_r_debug_t debug; + kzt_x86_64_r_debug_t final_debug; + uintptr_t current; + uintptr_t previous = 0; + size_t count = 0; + + if (!observer || !observer->active || !observer->r_debug_addr || + !object_name || !symbol_name || !address) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + *address = 0; + if (kzt_public_loader_read_guest( + observer->r_debug_addr, &debug, sizeof(debug), NULL) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (debug.version < 1 || + debug.state != KZT_LOADER_DEBUG_CONSISTENT || + !debug.map || !debug.brk || + (observer->r_brk_addr && + observer->r_brk_addr != (uintptr_t)debug.brk)) { + return debug.state == KZT_LOADER_DEBUG_ADD || + debug.state == KZT_LOADER_DEBUG_DELETE + ? KZT_PUBLIC_LOADER_BUSY + : KZT_PUBLIC_LOADER_INVALID_STATE; + } + + current = (uintptr_t)debug.map; + while (current) { + kzt_public_loader_object_t object; + kzt_public_loader_result_t result; + uintptr_t candidate = 0; + char name[KZT_GUEST_OBJECT_NAME_MAX]; + + if (++count > UINT16_MAX) { + return KZT_PUBLIC_LOADER_LIMIT; + } + result = kzt_read_guest_loader_object(current, &object); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + if (object.previous_addr != previous) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + if (object.name_addr) { + result = kzt_read_guest_object_name( + object.name_addr, name, sizeof(name)); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + if (kzt_guest_object_name_matches(name, object_name)) { + result = kzt_public_loader_find_symbol_in_object( + &object, &kzt_public_loader_reader, + symbol_name, &candidate); + if (result != KZT_PUBLIC_LOADER_OK && + result != KZT_PUBLIC_LOADER_NOT_FOUND) { + return result; + } + if (result == KZT_PUBLIC_LOADER_OK) { + if (*address && *address != candidate) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + *address = candidate; + } + } + } + previous = current; + current = object.next_addr; + } + if (kzt_public_loader_read_guest( + observer->r_debug_addr, &final_debug, + sizeof(final_debug), NULL) != 0) { + return KZT_PUBLIC_LOADER_READ_ERROR; + } + if (memcmp(&debug, &final_debug, sizeof(debug)) != 0) { + return KZT_PUBLIC_LOADER_BUSY; + } + return *address ? KZT_PUBLIC_LOADER_OK + : KZT_PUBLIC_LOADER_NOT_FOUND; +} + +uintptr_t kzt_resolve_guest_object_symbol( + const char *object_name, + const char *symbol_name) +{ + enum { KZT_GUEST_OBJECT_NAME_MAX = 512 }; + kzt_public_loader_observer_t snapshot; + kzt_public_loader_result_t result; + uintptr_t address = 0; + uintptr_t dynamic_addr = 0; + size_t dynamic_count = 0; + + if (!object_name || !symbol_name) { + return 0; + } + mmap_lock(); + snapshot = kzt_public_loader_observer; + if (snapshot.active) { + result = kzt_public_loader_observer_refresh( + &snapshot, &kzt_public_loader_reader, + kzt_loader_snapshot_visit, NULL); + } else if (kzt_find_main_dynamic_table( + elf_header, &dynamic_addr, &dynamic_count) == 0) { + result = kzt_public_loader_observer_activate( + &snapshot, dynamic_addr, dynamic_count, + &kzt_public_loader_reader, kzt_loader_snapshot_visit, NULL); + } else { + result = KZT_PUBLIC_LOADER_NOT_FOUND; + } + if (result == KZT_PUBLIC_LOADER_LIMIT) { + result = kzt_resolve_guest_object_symbol_live( + &snapshot, object_name, symbol_name, &address); + goto out; + } + if (result != KZT_PUBLIC_LOADER_OK) { + goto out; + } + for (size_t index = 0; index < snapshot.live_map_count; ++index) { + kzt_public_loader_object_t object; + uintptr_t candidate = 0; + char name[KZT_GUEST_OBJECT_NAME_MAX]; + + result = kzt_read_guest_loader_object( + snapshot.live_maps[index], &object); + if (result != KZT_PUBLIC_LOADER_OK) { + goto out; + } + if (!object.name_addr) { + continue; + } + result = kzt_read_guest_object_name( + object.name_addr, name, sizeof(name)); + if (result != KZT_PUBLIC_LOADER_OK) { + goto out; + } + if (!kzt_guest_object_name_matches(name, object_name)) { + continue; + } + result = kzt_public_loader_find_symbol_in_object( + &object, &kzt_public_loader_reader, + symbol_name, &candidate); + if (result == KZT_PUBLIC_LOADER_NOT_FOUND) { + continue; + } + if (result != KZT_PUBLIC_LOADER_OK || + (address && address != candidate)) { + address = 0; + goto out; + } + address = candidate; + } + +out: + mmap_unlock(); + return address; +} + +uintptr_t kzt_resolve_guest_link_map_symbol( + uintptr_t link_map_addr, + const char *symbol_name) +{ + kzt_public_loader_object_t object; + kzt_public_loader_result_t result; + uintptr_t address = 0; + + if (!link_map_addr || !symbol_name) { + return 0; + } + mmap_lock(); + result = kzt_read_guest_loader_object(link_map_addr, &object); + if (result == KZT_PUBLIC_LOADER_OK) { + result = kzt_public_loader_find_symbol_in_object( + &object, &kzt_public_loader_reader, + symbol_name, &address); + } + mmap_unlock(); + return result == KZT_PUBLIC_LOADER_OK ? address : 0; +} + +static int kzt_main_elf_contains_address(uintptr_t guest_addr) +{ + uintptr_t load_bias; + + if (!elf_header || !elf_header->PHEntries) { + return 0; + } + load_bias = (uintptr_t)elf_header->delta; + for (size_t index = 0; index < elf_header->numPHEntries; ++index) { + const Elf64_Phdr *phdr = &elf_header->PHEntries[index]; + uintptr_t segment_start; + uintptr_t segment_end; + + if (phdr->p_type != PT_LOAD || !phdr->p_memsz || + phdr->p_vaddr > UINTPTR_MAX - load_bias) { + continue; + } + segment_start = load_bias + (uintptr_t)phdr->p_vaddr; + if (phdr->p_memsz > UINTPTR_MAX - segment_start) { + continue; + } + segment_end = segment_start + (uintptr_t)phdr->p_memsz; + if (guest_addr >= segment_start && guest_addr < segment_end) { + return 1; + } + } + return 0; +} + +static kzt_public_loader_result_t kzt_find_zero_bias_main_object( + const kzt_public_loader_observer_t *observer, + uintptr_t guest_addr, + kzt_public_loader_object_t *object) +{ + kzt_public_loader_object_t candidate = { 0 }; + int found = 0; + + if (!kzt_main_elf_contains_address(guest_addr)) { + return KZT_PUBLIC_LOADER_NOT_FOUND; + } + for (size_t index = 0; index < observer->live_map_count; ++index) { + kzt_public_loader_object_t current; + kzt_public_loader_result_t result = + kzt_read_guest_loader_object( + observer->live_maps[index], ¤t); + + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + if (current.load_bias) { + continue; + } + if (found) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + candidate = current; + found = 1; + } + if (!found) { + return KZT_PUBLIC_LOADER_NOT_FOUND; + } + *object = candidate; + return KZT_PUBLIC_LOADER_OK; +} + +static kzt_public_loader_result_t kzt_find_external_object_by_address( + uintptr_t guest_addr, + kzt_public_loader_object_t *object) +{ + kzt_public_loader_object_t candidate = { 0 }; + int found = 0; + + for (size_t index = 0; + index < kzt_guest_tls_external_map_count; ++index) { + kzt_public_loader_object_t current; + kzt_public_loader_result_t result = + kzt_read_guest_loader_object( + kzt_guest_tls_external_maps[index].link_map_addr, + ¤t); + int contains; + + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + result = kzt_public_loader_object_contains_address( + ¤t, &kzt_public_loader_reader, + guest_addr, &contains); + if (result != KZT_PUBLIC_LOADER_OK) { + return result; + } + if (!contains) { + continue; + } + if (found && candidate.link_map_addr != current.link_map_addr) { + return KZT_PUBLIC_LOADER_INVALID_STATE; + } + candidate = current; + found = 1; + } + if (!found) { + return KZT_PUBLIC_LOADER_NOT_FOUND; + } + *object = candidate; + return KZT_PUBLIC_LOADER_OK; +} + +static kzt_public_loader_result_t kzt_find_guest_object_by_address( + uintptr_t guest_addr, + kzt_public_loader_object_t *object) +{ + kzt_public_loader_observer_t snapshot; + kzt_public_loader_object_t public_object = { 0 }; + kzt_public_loader_object_t main_object = { 0 }; + kzt_public_loader_object_t external_object = { 0 }; + kzt_public_loader_result_t result; + kzt_public_loader_result_t refresh_result; + kzt_public_loader_result_t main_result; + kzt_public_loader_result_t external_result; + uintptr_t dynamic_addr = 0; + size_t dynamic_count = 0; + + if (!guest_addr || !object) { + return KZT_PUBLIC_LOADER_INVALID_INPUT; + } + memset(object, 0, sizeof(*object)); + + mmap_lock(); + snapshot = kzt_public_loader_observer; + if (snapshot.active) { + refresh_result = kzt_public_loader_observer_refresh( + &snapshot, &kzt_public_loader_reader, + kzt_loader_snapshot_visit, NULL); + } else if (kzt_find_main_dynamic_table( + elf_header, &dynamic_addr, &dynamic_count) == 0) { + refresh_result = kzt_public_loader_observer_activate( + &snapshot, dynamic_addr, dynamic_count, + &kzt_public_loader_reader, kzt_loader_snapshot_visit, NULL); + } else { + refresh_result = KZT_PUBLIC_LOADER_NOT_FOUND; + } + result = refresh_result; + if (refresh_result == KZT_PUBLIC_LOADER_OK || + refresh_result == KZT_PUBLIC_LOADER_BUSY || + refresh_result == KZT_PUBLIC_LOADER_LIMIT) { + result = kzt_public_loader_find_object_by_address( + &snapshot, &kzt_public_loader_reader, + guest_addr, &public_object); + } + if (result != KZT_PUBLIC_LOADER_OK && + result != KZT_PUBLIC_LOADER_NOT_FOUND && + result != KZT_PUBLIC_LOADER_BUSY && + result != KZT_PUBLIC_LOADER_LIMIT) { + goto out; + } + main_result = kzt_find_zero_bias_main_object( + &snapshot, guest_addr, &main_object); + if (main_result != KZT_PUBLIC_LOADER_OK && + main_result != KZT_PUBLIC_LOADER_NOT_FOUND) { + result = main_result; + goto out; + } + external_result = kzt_find_external_object_by_address( + guest_addr, &external_object); + if (external_result != KZT_PUBLIC_LOADER_OK && + external_result != KZT_PUBLIC_LOADER_NOT_FOUND) { + result = external_result; + goto out; + } + if ((result == KZT_PUBLIC_LOADER_OK && + main_result == KZT_PUBLIC_LOADER_OK && + public_object.link_map_addr != main_object.link_map_addr) || + (result == KZT_PUBLIC_LOADER_OK && + external_result == KZT_PUBLIC_LOADER_OK && + public_object.link_map_addr != external_object.link_map_addr) || + (main_result == KZT_PUBLIC_LOADER_OK && + external_result == KZT_PUBLIC_LOADER_OK && + main_object.link_map_addr != external_object.link_map_addr)) { + result = KZT_PUBLIC_LOADER_INVALID_STATE; + goto out; + } + if (result == KZT_PUBLIC_LOADER_OK) { + *object = public_object; + } else if (main_result == KZT_PUBLIC_LOADER_OK) { + *object = main_object; + result = KZT_PUBLIC_LOADER_OK; + } else if (external_result == KZT_PUBLIC_LOADER_OK) { + *object = external_object; + result = KZT_PUBLIC_LOADER_OK; + } +out: + mmap_unlock(); + return result; +} + +uintptr_t kzt_find_guest_link_map_by_address_ex( + uintptr_t guest_addr, + kzt_public_loader_result_t *lookup_result) +{ + kzt_public_loader_object_t object; + kzt_public_loader_result_t result; + + result = kzt_find_guest_object_by_address(guest_addr, &object); + if (lookup_result) { + *lookup_result = result; + } + if (result != KZT_PUBLIC_LOADER_OK) { + printf_log(LOG_DEBUG, + "KZT cannot resolve Guest address %p to link_map: %s\n", + (void *)guest_addr, + kzt_public_loader_result_name(result)); + return 0; + } + return object.link_map_addr; +} + +uintptr_t kzt_find_guest_link_map_by_address(uintptr_t guest_addr) +{ + return kzt_find_guest_link_map_by_address_ex(guest_addr, NULL); +} + +int kzt_collect_guest_tls_objects( + kzt_public_loader_tls_object_t *objects, + size_t object_capacity, + size_t *object_count) +{ + kzt_public_loader_result_t result; + uintptr_t dynamic_addr = 0; + size_t dynamic_count = 0; + + mmap_lock(); + if (kzt_guest_tls_external_error) { + mmap_unlock(); + return -1; + } + if (!kzt_public_loader_observer.active) { + if (kzt_find_main_dynamic_table( + elf_header, &dynamic_addr, &dynamic_count) != 0) { + mmap_unlock(); + return -1; + } + } + result = kzt_public_loader_snapshot_tls( + &kzt_public_loader_observer, dynamic_addr, dynamic_count, + &kzt_public_loader_reader, 0, + objects, object_capacity, object_count); + if (result == KZT_PUBLIC_LOADER_OK) { + for (size_t external_index = 0; + external_index < kzt_guest_tls_external_map_count; + ++external_index) { + kzt_x86_64_link_map_prefix_t map; + kzt_public_loader_object_t object; + kzt_public_loader_tls_object_t tls_object; + int duplicate = 0; + int has_tls = 0; + + for (size_t object_index = 0; + object_index < *object_count; ++object_index) { + if (objects[object_index].link_map_addr == + kzt_guest_tls_external_maps[external_index] + .link_map_addr) { + duplicate = 1; + break; + } + } + if (duplicate) { + continue; + } + if (kzt_public_loader_read_guest( + kzt_guest_tls_external_maps[external_index] + .link_map_addr, + &map, sizeof(map), NULL) != 0) { + result = KZT_PUBLIC_LOADER_READ_ERROR; + break; + } + object = (kzt_public_loader_object_t) { + .link_map_addr = + kzt_guest_tls_external_maps[external_index] + .link_map_addr, + .load_bias = (uintptr_t)map.load_bias, + .name_addr = (uintptr_t)map.name, + .dynamic_addr = (uintptr_t)map.dynamic_addr, + .next_addr = (uintptr_t)map.next, + .previous_addr = (uintptr_t)map.previous, + }; + result = kzt_public_loader_read_tls_object( + &object, &kzt_public_loader_reader, + &tls_object, &has_tls); + if (result != KZT_PUBLIC_LOADER_OK) { + break; + } + if (!has_tls) { + continue; + } + if (*object_count == object_capacity) { + result = KZT_PUBLIC_LOADER_LIMIT; + break; + } + tls_object.load_generation = + kzt_guest_tls_external_maps[external_index] + .load_generation; + tls_object.external_registration = 1; + objects[(*object_count)++] = tls_object; + } + } + mmap_unlock(); + if (result == KZT_PUBLIC_LOADER_OK) { + return 0; + } + if (result != KZT_PUBLIC_LOADER_BUSY) { + fprintf(stderr, + "KZT Guest TLS collection failed: %s\n", + kzt_public_loader_result_name(result)); + } + return result == KZT_PUBLIC_LOADER_BUSY + ? KZT_GUEST_TLS_REFRESH_BUSY : -1; +} + +int kzt_collect_guest_tls_object( + uintptr_t link_map_addr, + kzt_public_loader_tls_object_t *object, + int *has_tls) +{ + kzt_public_loader_result_t result; + uintptr_t dynamic_addr = 0; + size_t dynamic_count = 0; + size_t object_count = 0; + + if (!link_map_addr || !object || !has_tls) { + return -1; + } + *has_tls = 0; + mmap_lock(); + if (kzt_guest_tls_external_error) { + mmap_unlock(); + return -1; + } + if (!kzt_public_loader_observer.active && + kzt_find_main_dynamic_table( + elf_header, &dynamic_addr, &dynamic_count) != 0) { + mmap_unlock(); + return -1; + } + result = kzt_public_loader_snapshot_tls( + &kzt_public_loader_observer, dynamic_addr, dynamic_count, + &kzt_public_loader_reader, link_map_addr, + object, 1, &object_count); + mmap_unlock(); + if (result == KZT_PUBLIC_LOADER_OK) { + *has_tls = object_count == 1; + return 0; + } + return result == KZT_PUBLIC_LOADER_BUSY + ? KZT_GUEST_TLS_REFRESH_BUSY : -1; +} + +int kzt_materialize_guest_tls_image( + const kzt_public_loader_tls_object_t *object, + void *destination, + size_t destination_size) +{ + return kzt_public_loader_materialize_tls_image( + object, &kzt_public_loader_reader, + destination, destination_size) == KZT_PUBLIC_LOADER_OK + ? 0 : -1; +} + +int kzt_register_guest_tls_link_map(uintptr_t link_map_addr) +{ + if (!latx_kzt_guest_tls_enabled()) { + return 0; + } + int result = -1; + + if (!link_map_addr) { + return -1; + } + mmap_lock(); + for (size_t index = 0; + index < kzt_guest_tls_external_map_count; ++index) { + if (kzt_guest_tls_external_maps[index].link_map_addr == + link_map_addr) { + if (kzt_guest_tls_external_maps[index].references == + SIZE_MAX) { + kzt_guest_tls_external_error = 1; + goto out; + } + ++kzt_guest_tls_external_maps[index].references; + result = 0; + goto out; + } + } + if (kzt_guest_tls_external_map_count == + KZT_PUBLIC_LOADER_MAX_OBJECTS || + kzt_guest_tls_external_generation == UINT64_MAX) { + kzt_guest_tls_external_error = 1; + goto out; + } + kzt_guest_tls_external_maps[kzt_guest_tls_external_map_count++] = + (kzt_guest_tls_external_map_t) { + .link_map_addr = link_map_addr, + .load_generation = ++kzt_guest_tls_external_generation, + .references = 1, + }; + result = 0; + +out: + mmap_unlock(); + return result; +} + +void kzt_unregister_guest_tls_link_map(uintptr_t link_map_addr) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + mmap_lock(); + for (size_t index = 0; + index < kzt_guest_tls_external_map_count; ++index) { + if (kzt_guest_tls_external_maps[index].link_map_addr != + link_map_addr) { + continue; + } + if (--kzt_guest_tls_external_maps[index].references != 0) { + break; + } + memmove(&kzt_guest_tls_external_maps[index], + &kzt_guest_tls_external_maps[index + 1], + (kzt_guest_tls_external_map_count - index - 1) * + sizeof(kzt_guest_tls_external_maps[0])); + --kzt_guest_tls_external_map_count; + break; + } + mmap_unlock(); +} + static void kzt_report_object_fallback_once( const kzt_public_loader_object_t *object, const char *name, @@ -2385,6 +3112,10 @@ uintptr_t kzt_resolve_guest_symbol(const char *name) return 0; } mmap_lock(); + if (latx_kzt_guest_tls_enabled() && kzt_guest_tls_external_error) { + mmap_unlock(); + return 0; + } snapshot = kzt_public_loader_observer; if (snapshot.active) { result = kzt_public_loader_observer_refresh( @@ -2597,12 +3328,6 @@ static int kzt_try_bind_loaded_object( rfilename = kzt_find_realsofilepath(rfilename, filetmp); } - printf_log(LOG_DEBUG, - "%d debug %s link_map=%p{0x%lx, %s, l_ld=%p}\n", - getpid(), __func__, (void *)object->link_map_addr, - object->load_bias, name_copy, - (void *)object->dynamic_addr); - f = fopen(rfilename, "rb"); if (!f) { kzt_report_object_fallback_once( @@ -2625,6 +3350,13 @@ static int kzt_try_bind_loaded_object( kzt_calculate_loaded_elf_range(h, object->load_bias, &map_start, &map_end); + printf_log(LOG_DEBUG, + "%d debug %s link_map=%p{load=[%p,%p), bias=%p, " + "%s, l_ld=%p}\n", + getpid(), __func__, (void *)object->link_map_addr, + (void *)map_start, (void *)map_end, + (void *)object->load_bias, name_copy, + (void *)object->dynamic_addr); ElfHeadReFix(h, object->load_bias); if (!have_mmap_lock() || !KZTRelocationTargetsAreWritable(h)) { @@ -2843,6 +3575,19 @@ void kzt_try_bind_before_guest_relro(uintptr_t start, size_t length, int prot) "mprotect(%p, %zu)\n", (void *)object.link_map_addr, (void *)start, length); } + if (latx_kzt_guest_tls_enabled() && + result == KZT_PUBLIC_LOADER_OK && lsenv && lsenv->cpu_state) { + CPUX86State *env = (CPUX86State *)lsenv->cpu_state; + + if (env->kzt_guest_tls_allocation && + kzt_guest_tls_preinitialize_static( + env, object.link_map_addr) != 0) { + fprintf(stderr, + "KZT cannot initialize static Guest TLS before " + "RELRO protection; refusing to run constructors\n"); + _exit(EXIT_FAILURE); + } + } in_preprotect = 0; } @@ -2965,18 +3710,49 @@ static void kzt_dynamic_library_change_callback(CPUX86State *env) { kzt_public_loader_result_t result; - (void)env; + kzt_guest_tls_loader_event_begin(); mmap_lock(); result = kzt_public_loader_observer_refresh( &kzt_public_loader_observer, &kzt_public_loader_reader, kzt_try_bind_observed_object, NULL); mmap_unlock(); if (result != KZT_PUBLIC_LOADER_OK && - result != KZT_PUBLIC_LOADER_BUSY) { + result != KZT_PUBLIC_LOADER_BUSY && + result != KZT_PUBLIC_LOADER_LIMIT) { kzt_report_observer_failure_once("refresh", result); printf_log(LOG_INFO, "KZT public loader refresh failed: %s\n", kzt_public_loader_result_name(result)); + if (env && env->kzt_guest_tls_allocation) { + fprintf(stderr, + "KZT public loader state is invalid before " + "dynamic-library constructors; refusing to continue\n"); + _exit(EXIT_FAILURE); + } + } + if (latx_kzt_guest_tls_enabled() && + (result == KZT_PUBLIC_LOADER_OK || + result == KZT_PUBLIC_LOADER_LIMIT)) { + int event_result = kzt_guest_tls_loader_event_observe(); + + if (event_result < 0) { + fprintf(stderr, + "KZT cannot observe Guest TLS loader generation; " + "refusing to continue\n"); + _exit(EXIT_FAILURE); + } + int tls_result = kzt_guest_tls_refresh_local(env); + + if (tls_result == KZT_GUEST_TLS_REFRESH_BUSY) { + return; + } + if (tls_result == 0) { + return; + } + fprintf(stderr, + "KZT cannot refresh attached Guest TLS before " + "dynamic-library constructors; refusing to continue\n"); + _exit(EXIT_FAILURE); } } @@ -2987,7 +3763,21 @@ static void kzt_guest_main_entry_callback(CPUX86State *env) uintptr_t dynamic_addr = 0; size_t dynamic_count = 0; + kzt_guest_tls_loader_tracking_reset(); + if (latx_finalize_host_thread_template(env) != 0) { + fprintf(stderr, + "KZT Host-thread template initialization failed; " + "refusing native bindings\n"); + exit(EXIT_FAILURE); + } mmap_lock(); + if (latx_kzt_guest_tls_enabled()) { + memset(kzt_guest_tls_external_maps, 0, + sizeof(kzt_guest_tls_external_maps)); + kzt_guest_tls_external_map_count = 0; + kzt_guest_tls_external_generation = 0; + kzt_guest_tls_external_error = 0; + } if (kzt_find_main_dynamic_table( elf_header, &dynamic_addr, &dynamic_count) == 0) { observer_result = kzt_public_loader_observer_activate( @@ -3018,11 +3808,13 @@ static void kzt_guest_main_entry_callback(CPUX86State *env) CPUState *cpu = env_cpu(env); target_ulong eip = env->eip; + kzt_guest_tls_loader_tracking_enable(); kzt_install_guest_pc_callback( kzt_public_r_brk_inst, cpu, kzt_public_loader_observer.r_brk_addr, kzt_dynamic_library_change_callback); env->eip = eip; + kzt_install_guest_fork_callback(cpu); printf_log(LOG_INFO, "KZT public loader observer active: " "r_debug=%p r_brk=%p objects=%zu\n", @@ -3042,6 +3834,36 @@ static void kzt_guest_main_entry_callback(CPUX86State *env) x64free_fini = 1; AddDebugInfo(LIB_EMULATED, elf_header->name, info1.start_code, info1.end_code); } + +static void kzt_guest_fork_entry_callback(CPUX86State *env) +{ + (void)env; + kzt_guest_tls_fork_prepare_early(); +} +static void kzt_install_guest_fork_callback(CPUState *cpu) +{ + CPUX86State *env = cpu->env_ptr; + target_ulong eip = env->eip; + + if (!latx_kzt_guest_tls_enabled()) { + return; + } + if (!kzt_guest_fork_addr) { + /* libpthread and libc can both export fork at different addresses. */ + kzt_guest_fork_addr = kzt_resolve_guest_object_symbol("libc.so.6", "fork"); + if (!kzt_guest_fork_addr) { + kzt_guest_fork_addr = kzt_resolve_guest_symbol("__libc_fork"); + } + } + if (kzt_guest_fork_addr) { + /* A full flush invalidates the saved translated instructions too. */ + memset(kzt_guest_fork_inst, 0, sizeof(kzt_guest_fork_inst)); + kzt_install_guest_pc_callback(kzt_guest_fork_inst, cpu, + kzt_guest_fork_addr, kzt_guest_fork_entry_callback); + } + env->eip = eip; +} + static TranslationBlock *kzt_install_guest_pc_callback( uint32_t *inst_old, CPUState *cpu, uintptr_t addr, void (*callback)(CPUX86State *)) @@ -3129,11 +3951,31 @@ void kzt_install_runtime_callbacks(CPUState *cpu, void *info) env = cpu->env_ptr; eip = env->eip; + if (latx_kzt_guest_tls_enabled() && + kzt_public_loader_observer.active && + kzt_public_loader_observer.r_brk_addr) { + memset(kzt_public_r_brk_inst, 0, + sizeof(kzt_public_r_brk_inst)); + kzt_install_guest_pc_callback( + kzt_public_r_brk_inst, cpu, + kzt_public_loader_observer.r_brk_addr, + kzt_dynamic_library_change_callback); + kzt_install_guest_fork_callback(cpu); + env->eip = eip; + return; + } + + kzt_guest_tls_loader_tracking_reset(); kzt_public_loader_observer_reset(&kzt_public_loader_observer); memset(kzt_public_r_brk_inst, 0, sizeof(kzt_public_r_brk_inst)); + memset(kzt_guest_fork_inst, 0, sizeof(kzt_guest_fork_inst)); + kzt_guest_fork_addr = 0; kzt_main_relocated_before_relro = 0; kzt_main_fallback_reported = 0; kzt_observer_failure_reported = 0; + if (latx_kzt_guest_tls_enabled()) { + memset(jmpinst_exec, 0, sizeof(jmpinst_exec)); + } /* * The program-entry hook is version independent. It installs the @@ -3267,13 +4109,12 @@ void kzt_wine_init_x86(void) return; } struct malloc_map* m = malloc(sizeof(struct malloc_map)); - m->mallocp = (void *)(uintptr_t)RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dlsym, 2, - 0, "malloc"); - ; - m->freep = (void *)(uintptr_t)RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dlsym, 2, - 0, "free"); - m->reallocp = (void *)(uintptr_t)RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dlsym, 2, - 0, "realloc"); + m->mallocp = (void *)(uintptr_t)RunFunctionWithStateInternal( + (uintptr_t)my_context->dlprivate->x86dlsym, 2, 0, "malloc"); + m->freep = (void *)(uintptr_t)RunFunctionWithStateInternal( + (uintptr_t)my_context->dlprivate->x86dlsym, 2, 0, "free"); + m->reallocp = (void *)(uintptr_t)RunFunctionWithStateInternal( + (uintptr_t)my_context->dlprivate->x86dlsym, 2, 0, "realloc"); m->h = wine_elf_header; AddMallocMap(my_context, m); x86free = m->freep; diff --git a/target/i386/latx/context/wrappedlibc.c b/target/i386/latx/context/wrappedlibc.c index 7fa2c9ac09..bcafdd150c 100644 --- a/target/i386/latx/context/wrappedlibc.c +++ b/target/i386/latx/context/wrappedlibc.c @@ -70,6 +70,8 @@ #include "bridge.h" #include "globalsymbols.h" #include "x86dlfun.h" +#include "kzt-guest-tls.h" +#include "kzt-guest-thread.h" #define LIBNAME libc const char* libcName = @@ -2337,16 +2339,13 @@ EXPORT int32_t my_posix_spawnp(pid_t* pid, const char* path, } EXPORT void my__Jv_RegisterClasses(void) {} -#if 0 EXPORT int32_t my___cxa_thread_atexit_impl(void* dtor, void* obj, void* dso) { - //printf_log(LOG_INFO, "Warning, call to __cxa_thread_atexit_impl(%p, %p, %p) ignored\n", dtor, obj, dso); - AddCleanup1Arg(dtor, obj, dso); - return 0; + __MY_CPU; - return 0; + return kzt_guest_thread_cxa_atexit( + cpu, (uintptr_t)dtor, (uintptr_t)obj, (uintptr_t)dso); } -#endif EXPORT int32_t my___register_atfork(void* prepare, void* parent, void* child, void* handle) { // this is partly incorrect, because the emulated funcionts should be executed by actual fork and not by my_atfork... @@ -3382,6 +3381,28 @@ EXPORT int my_register_printf_type(void* f) return my->register_printf_type(findprintf_typeFct(f)); } + + +EXPORT int my_pthread_key_create(unsigned int *key, void *destructor) +{ + __MY_CPU; + + return kzt_guest_thread_key_create( + cpu, key, (uintptr_t)destructor); +} + +EXPORT int my_pthread_key_delete(unsigned int key) +{ + __MY_CPU; + + return kzt_guest_thread_key_delete(cpu, key); +} + +EXPORT int my___pthread_key_create(unsigned int *key, void *destructor) +{ + return my_pthread_key_create(key, destructor); +} + EXPORT void my___libc_free(void* m) { lsassert(0);//for translate_free_int3 @@ -3493,4 +3514,10 @@ int box64_isglibc234 = 1; #pragma GCC diagnostic pop +#define WRAPPEDLIB_FUNCTION_ENABLED(name) \ + (latx_kzt_guest_tls_enabled() || \ + (strcmp(name, "__cxa_thread_atexit_impl") && \ + strcmp(name, "pthread_key_create") && \ + strcmp(name, "__pthread_key_create") && \ + strcmp(name, "pthread_key_delete"))) #include "wrappedlib_init.h" diff --git a/target/i386/latx/context/wrappedlibdl.c b/target/i386/latx/context/wrappedlibdl.c index 95a246880e..458db12978 100644 --- a/target/i386/latx/context/wrappedlibdl.c +++ b/target/i386/latx/context/wrappedlibdl.c @@ -28,8 +28,10 @@ #include "elfloader.h" #include "elfloader_private.h" #include "callback.h" +#include "kzt-guest-tls.h" #include "myalign.h" #include "fileutils.h" +#include "qemu.h" #include "x86dlfun.h" #ifndef CONFIG_LOONGARCH_NEW_WORLD @@ -38,6 +40,147 @@ const char *libdlName = "libdl.so.2"; #endif #define FORWORDBACK 0 + +#ifdef CONFIG_LATX_KZT +static GRecMutex kzt_guest_loader_operation_lock; +static GMutex kzt_dl_metadata_lock; +static gsize kzt_guest_loader_operation_lock_initialized; +static GThread *kzt_guest_loader_operation_owner; +static gint kzt_guest_loader_operation_depth; + +int kzt_guest_loader_operation_active(void) +{ + return g_atomic_int_get(&kzt_guest_loader_operation_depth) != 0; +} + +static int kzt_guest_loader_operation_enter(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return 0; + } + CPUX86State *cpu = lsenv && lsenv->cpu_state + ? (CPUX86State *)lsenv->cpu_state : NULL; + int execution_paused = + kzt_guest_tls_execution_pause(cpu); + + if (g_once_init_enter( + &kzt_guest_loader_operation_lock_initialized)) { + g_rec_mutex_init(&kzt_guest_loader_operation_lock); + g_once_init_leave( + &kzt_guest_loader_operation_lock_initialized, 1); + } + g_rec_mutex_lock(&kzt_guest_loader_operation_lock); + g_assert(!g_atomic_int_get(&kzt_guest_loader_operation_depth) || + g_atomic_pointer_get( + &kzt_guest_loader_operation_owner) == g_thread_self()); + g_atomic_pointer_set( + &kzt_guest_loader_operation_owner, g_thread_self()); + g_atomic_int_inc(&kzt_guest_loader_operation_depth); + kzt_guest_tls_execution_resume(cpu, execution_paused); + return 1; +} + +static void kzt_guest_loader_operation_leave(int *locked) +{ + if (locked && *locked) { + g_assert(g_atomic_int_get( + &kzt_guest_loader_operation_depth) > 0 && + g_atomic_pointer_get( + &kzt_guest_loader_operation_owner) == + g_thread_self()); + if (g_atomic_int_dec_and_test( + &kzt_guest_loader_operation_depth)) { + g_atomic_pointer_set( + &kzt_guest_loader_operation_owner, NULL); + } + g_rec_mutex_unlock(&kzt_guest_loader_operation_lock); + } +} + +void kzt_guest_loader_after_fork_child(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + GThread *current = g_thread_self(); + int depth = g_atomic_pointer_get( + &kzt_guest_loader_operation_owner) == current + ? g_atomic_int_get(&kzt_guest_loader_operation_depth) : 0; + + memset(&kzt_guest_loader_operation_lock, 0, + sizeof(kzt_guest_loader_operation_lock)); + g_rec_mutex_init(&kzt_guest_loader_operation_lock); + memset(&kzt_dl_metadata_lock, 0, + sizeof(kzt_dl_metadata_lock)); + g_mutex_init(&kzt_dl_metadata_lock); + kzt_guest_loader_operation_lock_initialized = 1; + g_atomic_pointer_set(&kzt_guest_loader_operation_owner, + depth ? current : NULL); + g_atomic_int_set(&kzt_guest_loader_operation_depth, 0); + for (int index = 0; index < depth; ++index) { + g_rec_mutex_lock(&kzt_guest_loader_operation_lock); + g_atomic_int_inc(&kzt_guest_loader_operation_depth); + } +} + +#define KZT_GUEST_LOADER_OPERATION_GUARD() \ + int kzt_guest_loader_operation_guard \ + __attribute__((cleanup(kzt_guest_loader_operation_leave), \ + unused)) = \ + kzt_guest_loader_operation_enter() + +static void kzt_dl_metadata_lock_acquire(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + g_mutex_lock(&kzt_dl_metadata_lock); +} + +static void kzt_dl_metadata_lock_release(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + g_mutex_unlock(&kzt_dl_metadata_lock); +} +#else +#define KZT_GUEST_LOADER_OPERATION_GUARD() do { } while (0) +static void kzt_dl_metadata_lock_acquire(void) +{ +} + +static void kzt_dl_metadata_lock_release(void) +{ +} +#endif + +#ifdef CONFIG_LATX_KZT +uintptr_t kzt_guest_loader_hold_open(uintptr_t guest_name) +{ + KZT_GUEST_LOADER_OPERATION_GUARD(); + dlprivate_t *dl = my_context ? my_context->dlprivate : NULL; + + if (!guest_name || !dl || !dl->x86dlopen) { + return 0; + } + return RunFunctionWithStateInternalNoRefresh( + (uintptr_t)dl->x86dlopen, 2, + guest_name, RTLD_LAZY | RTLD_NOLOAD); +} + +void kzt_guest_loader_hold_close(uintptr_t handle) +{ + KZT_GUEST_LOADER_OPERATION_GUARD(); + dlprivate_t *dl = my_context ? my_context->dlprivate : NULL; + + if (handle && dl && dl->x86dlclose) { + (void)RunFunctionWithStateInternalNoRefresh( + (uintptr_t)dl->x86dlclose, 1, handle); + } +} +#endif + dlprivate_t *NewDLPrivate(void) { dlprivate_t* dl = (dlprivate_t*)box_calloc(1, sizeof(dlprivate_t)); return dl; @@ -52,7 +195,7 @@ static __thread int dl_error_pending; static void clear_dl_error(dlprivate_t *dl) { if (dl && dl->x86dlerror) - (void)RunFunctionWithState((uintptr_t)dl->x86dlerror, 0); + (void)RunFunctionWithStateInternal((uintptr_t)dl->x86dlerror, 0); dl_error_pending = 0; } @@ -74,6 +217,77 @@ static void set_dl_errorf(dlprivate_t *dl, const char *format, ...) set_dl_error(dl, message); } +static void *finish_dlopen_with_guest_tls(dlprivate_t *dl, void *result) +{ +#ifdef CONFIG_LATX_KZT + if (!latx_kzt_guest_tls_enabled()) { + return result; + } + if (result) { + uintptr_t link_map_addr = (uintptr_t)result; + __MY_CPU; + + if (link_map_addr <= dl->lib_sz && link_map_addr != 0) { + library_t *lib = dl->libs[link_map_addr - 1]; + + link_map_addr = lib ? (uintptr_t)lib->x86linkmap : 0; + } + if (link_map_addr && + kzt_register_guest_tls_link_map(link_map_addr) != 0) { + fprintf(stderr, + "KZT cannot register Guest TLS after a successful " + "dlopen; refusing to continue\n"); + _exit(EXIT_FAILURE); + } + + if (kzt_guest_tls_refresh(cpu) != 0) { + fprintf(stderr, + "KZT cannot refresh Guest TLS after a successful " + "dlopen; refusing to run further callbacks\n"); + _exit(EXIT_FAILURE); + } + } +#else + (void)dl; +#endif + return result; +} + +#ifdef CONFIG_LATX_KZT +static int call_guest_dlclose_with_guest_tls(void *handle) +{ + __MY_CPU; + int result; + + if (kzt_guest_tls_refresh(cpu) != 0) { + fprintf(stderr, + "KZT cannot refresh Guest TLS before dlclose; " + "refusing to continue\n"); + _exit(EXIT_FAILURE); + } + kzt_guest_tls_loader_event_begin(); + kzt_unregister_guest_tls_link_map((uintptr_t)handle); + result = (int)RunFunctionWithStateInternalNoRefresh( + (uintptr_t)my_context->dlprivate->x86dlclose, + 1, handle); + if (result == 0) { + if (kzt_guest_tls_refresh(cpu) != 0) { + fprintf(stderr, + "KZT cannot refresh Guest TLS after dlclose; " + "refusing to continue\n"); + _exit(EXIT_FAILURE); + } + } else if (kzt_register_guest_tls_link_map( + (uintptr_t)handle) != 0) { + fprintf(stderr, + "KZT cannot restore Guest TLS loader state after a " + "failed dlclose; refusing to continue\n"); + _exit(EXIT_FAILURE); + } + return result; +} +#endif + #define CLEARERR clear_dl_error(dl); static int replace_path_token(char **path, const char *token, @@ -159,11 +373,50 @@ static int init_x86dlfun(void) return init_x86dlfun_from("libdl.so.2", "libc.so.6"); #endif } + +static void *redlopen_guest_library(dlprivate_t *dl, library_t *lib) +{ + KZT_GUEST_LOADER_OPERATION_GUARD(); + __MY_CPU; + void *guest_handle; + + if (lib->x86linkmap) { + return lib->x86linkmap; + } + + if (kzt_guest_tls_refresh(cpu) != 0) { + return NULL; + } + kzt_guest_tls_loader_event_begin(); + guest_handle = (void *)(uintptr_t)RunFunctionWithStateInternalNoRefresh( + (uintptr_t)my_context->dlprivate->x86dlopen, 2, + lib->name, lib->x86dlopenflag); + if (!guest_handle) { + return NULL; + } + + kzt_dl_metadata_lock_acquire(); + lib->x86linkmap = guest_handle; + kzt_dl_metadata_lock_release(); + return finish_dlopen_with_guest_tls(dl, guest_handle); +} + static int callx86dlopen(void *filename, int flag, elfheader_t * h, int is_local) { - struct link_map* ret = (struct link_map*)(uintptr_t)RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dlopen, 2, filename, flag); + __MY_CPU; + + if (kzt_guest_tls_refresh(cpu) != 0) { + return -1; + } + kzt_guest_tls_loader_event_begin(); + struct link_map *ret = + (struct link_map *)(uintptr_t)RunFunctionWithStateInternalNoRefresh( + (uintptr_t)my_context->dlprivate->x86dlopen, 2, + filename, flag); if (ret) { printf_dlsym(LOG_DEBUG, "latx RunFunctionWithState dlopen %s addr %p\n", (char *)filename, (void *)ret->l_addr); + kzt_dl_metadata_lock_acquire(); h->lib->x86linkmap = ret; + kzt_dl_metadata_lock_release(); } else { //open error return -1; @@ -190,6 +443,7 @@ static void LatxResetElf(elfheader_t * h) h->latx_hasfix = 0; } EXPORT void* my_dlopen(void *filename, int flag){ + KZT_GUEST_LOADER_OPERATION_GUARD(); // TODO, handling special values for filename, like RTLD_SELF? // TODO, handling flags? library_t *lib = NULL; @@ -235,6 +489,12 @@ EXPORT void* my_dlopen(void *filename, int flag){ for (size_t i=0; ilib_sz; ++i) { if(IsSameLib(dl->libs[i], rfilename)) { if(dl->count[i]==0 && dl->dlopened[i]) { // need to lauch init again! + if (latx_kzt_guest_tls_enabled() && (flag & RTLD_NOLOAD)) { + box_free(rfilename); + set_dl_error( + dl, "RTLD_NOLOAD object is not loaded"); + return NULL; + } int idx = GetElfIndex(dl->libs[i]); if(idx!=-1) { printf_dlsym(LOG_DEBUG, "dlopen: Recycling, calling Init for %p (%s)\n", (void*)(i+1), rfilename); @@ -248,11 +508,15 @@ EXPORT void* my_dlopen(void *filename, int flag){ ReloadLibrary(dl->libs[i]); // reset memory image, redo reloc, run inits } } - if(!(flag&0x4)) - dl->count[i] = dl->count[i]+1; + kzt_dl_metadata_lock_acquire(); + if (latx_kzt_guest_tls_enabled() || !(flag & RTLD_NOLOAD)) { + dl->count[i] += 1; + } + kzt_dl_metadata_lock_release(); printf_dlsym(LOG_DEBUG, "dlopen: Recycling %s/%p count=%ld (dlopened=%ld, elf_index=%d)\n", rfilename, (void*)(i+1), dl->count[i], dl->dlopened[i], GetElfIndex(dl->libs[i])); box_free(rfilename); - return (void*)(i+1); + return finish_dlopen_with_guest_tls( + dl, (void *)(i + 1)); } } if(strstr(rfilename, "libGL.so")){ @@ -276,12 +540,21 @@ EXPORT void* my_dlopen(void *filename, int flag){ printf_dlsym(LOG_DEBUG, "warning call x86dlopen filename is %s %x\n", (char *)filename, flag); return NULL; #else - uint64_t ret = RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dlopen, 2, filename, flag); + __MY_CPU; + if (kzt_guest_tls_refresh(cpu) != 0) { + box_free(rfilename); + return NULL; + } + kzt_guest_tls_loader_event_begin(); + uint64_t ret = RunFunctionWithStateInternalNoRefresh( + (uintptr_t)my_context->dlprivate->x86dlopen, 2, + filename, flag); printf_dlsym(LOG_DEBUG, "warning call call x86dlopen filename %s %x ret=0x%lx\n", (char *)filename, flag, ret); //lsassert(0); if (ret) { box_free(rfilename); - return (void *)ret; + return finish_dlopen_with_guest_tls( + dl, (void *)(uintptr_t)ret); } set_dl_errorf(dl, "filename \"%s\" flag=%x\n", (char *)filename, flag); @@ -318,11 +591,16 @@ EXPORT void* my_dlopen(void *filename, int flag){ // check if already dlopenned... for (size_t i=0; ilib_sz; ++i) { if(!dl->libs[i]) { + kzt_dl_metadata_lock_acquire(); dl->count[i] = dl->count[i]+1; - return (void*)(i+1); + kzt_dl_metadata_lock_release(); + return finish_dlopen_with_guest_tls( + dl, (void *)(i + 1)); } } - printf_dlsym(LOG_DEBUG, "Call to dlopen(NULL, %X) forword call x86dlopen \n", flag); + printf_dlsym(LOG_DEBUG, + "Call to dlopen(NULL, %X), forward to x86 dlopen\n", + flag); lsassert(dl->x86dlopen); __MY_CPU; Push64(cpu, (uint64_t)dl->x86dlopen); @@ -330,6 +608,7 @@ EXPORT void* my_dlopen(void *filename, int flag){ } //get the lib and add it to the collection + kzt_dl_metadata_lock_acquire(); if(dl->lib_sz == dl->lib_cap) { dl->lib_cap += 4; dl->libs = (library_t**)box_realloc(dl->libs, sizeof(library_t*)*dl->lib_cap); @@ -342,15 +621,17 @@ EXPORT void* my_dlopen(void *filename, int flag){ dl->libs[idx] = lib; dl->count[idx] = dl->count[idx]+1; dl->dlopened[idx] = dlopened; + kzt_dl_metadata_lock_release(); printf_dlsym(LOG_DEBUG, "dlopen: New handle %p (%s), dlopened=%ld\n", (void*)(idx+1), (char*)filename, dlopened); if (lib && lib->type == LIB_EMULATED) { - return lib->x86linkmap; + return finish_dlopen_with_guest_tls(dl, lib->x86linkmap); } - return (void*)(idx+1); + return finish_dlopen_with_guest_tls(dl, (void *)(idx + 1)); } EXPORT void* my_dlmopen(void* lmid, void *filename, int flag) { + KZT_GUEST_LOADER_OPERATION_GUARD(); dlprivate_t *dl = my_context->dlprivate; if ((Lmid_t)lmid != LM_ID_BASE) { @@ -418,8 +699,31 @@ static int find_dl_library_index(dlprivate_t *dl, void *handle, size_t *index) return 0; } +typedef struct dl_handle_snapshot { + size_t index; + size_t count; + library_t *library; +} dl_handle_snapshot_t; + +static int snapshot_dl_handle(dlprivate_t *dl, void *handle, + dl_handle_snapshot_t *snapshot) +{ + int found; + + kzt_dl_metadata_lock_acquire(); + found = find_dl_library_index(dl, handle, &snapshot->index); + if (found) { + snapshot->count = dl->count[snapshot->index]; + snapshot->library = dl->libs[snapshot->index]; + } + kzt_dl_metadata_lock_release(); + return found; +} + EXPORT void* my_dlsym(void *handle, void *symbol){ dlprivate_t *dl = my_context->dlprivate; + dl_handle_snapshot_t handle_snapshot = { 0 }; + int known_handle = 0; uintptr_t start = 0, end = 0; char* rsymbol = (char*)symbol; CLEARERR @@ -431,10 +735,15 @@ EXPORT void* my_dlsym(void *handle, void *symbol){ } printf_dlsym(LOG_DEBUG, "Call to dlsym(%p, \"%s\")%s\n", handle, rsymbol, dlsym_error?"":"\n"); if (handle && handle != (void*)~0LL) { - size_t known_index; - if (!find_dl_library_index(dl, handle, &known_index)) { - uint64_t ret = RunFunctionWithState( + known_handle = snapshot_dl_handle( + dl, handle, &handle_snapshot); + if (!known_handle) { + uint64_t ret = RunFunctionWithStateInternal( (uintptr_t)dl->x86dlsym, 2, handle, symbol); + if (!ret) { + ret = kzt_resolve_guest_link_map_symbol( + (uintptr_t)handle, rsymbol); + } if (!ret) set_dl_errorf(dl, "Symbol \"%s\" not found in %p\n", rsymbol, handle); @@ -457,7 +766,9 @@ EXPORT void* my_dlsym(void *handle, void *symbol){ printf_dlsym(LOG_DEBUG, "warning call x86dlsym filename is NULL\n"); return NULL; #else - uint64_t ret = RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dlsym, 2, handle, symbol); + uint64_t ret = RunFunctionWithStateInternal( + (uintptr_t)my_context->dlprivate->x86dlsym, 2, + handle, symbol); printf_dlsym(LOG_DEBUG, "warning call x86dlsym filename is NULL ret=0x%lx\n", ret); if (ret) { return (void *)ret; @@ -481,18 +792,7 @@ EXPORT void* my_dlsym(void *handle, void *symbol){ printf_dlsym(LOG_DEBUG, "warning call x86dlsym filename is RTLD_NEXT\n"); return NULL; } - size_t nlib = (size_t)handle; - if(nlib > dl->lib_sz) { - for (int i = 0; i < dl->lib_sz; i++) { - if (dl->libs[i] && dl->libs[i]->active && dl->libs[i]->type == LIB_EMULATED && ((size_t)dl->libs[i]->x86linkmap) == nlib) { - nlib = i + 1; - break; - } - } - } - --nlib; - // size_t is unsigned - if(nlib>=dl->lib_sz) { + if (!known_handle) { #ifdef LATX_RELOCATION_SAVE_SYMBOLS if(GetGlobalSymbolStartEnd(my_context->maplib, rsymbol, &start, &end, NULL, -1, NULL)) { printf_dlsym(LOG_NEVER, "%p\n", (void*)start); @@ -538,7 +838,7 @@ EXPORT void* my_dlsym(void *handle, void *symbol){ printf_dlsym(LOG_DEBUG, "warning call x86dlsym filename is %s 0x%lx %s\n", strlen(lmfile)?lmfile:"NULL", cpu->regs[R_EDI], (char*)symbol); return NULL; #else - uint64_t ret = RunFunctionWithState( + uint64_t ret = RunFunctionWithStateInternal( (uintptr_t)my_context->dlprivate->x86dlsym, 2, handle, symbol); printf_dlsym(LOG_DEBUG, "warning call call x86dlsym filename is %s handle %p ret=0x%lx\n", strlen(lmfile)?lmfile:"NULL", handle, ret); @@ -550,45 +850,64 @@ EXPORT void* my_dlsym(void *handle, void *symbol){ return NULL; #endif } - if(dl->count[nlib]==0) { + if (handle_snapshot.count == 0) { set_dl_errorf(dl, "Bad handle %p (already closed))\n", handle); return NULL; } - if(dl->libs[nlib]) { - if(my_dlsym_lib(dl->libs[nlib], rsymbol, &start, &end, -1, NULL)==0) { + if (handle_snapshot.library) { + if (my_dlsym_lib(handle_snapshot.library, rsymbol, + &start, &end, -1, NULL) == 0) { // not found __MY_CPU; #if 1 - if(!dl->libs[nlib]->x86linkmap) { + if (!handle_snapshot.library->x86linkmap) { //redlopen - uint64_t ret = RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dlopen, 2, dl->libs[nlib]->name, dl->libs[nlib]->x86dlopenflag); - if (!ret) {//user sometime test for finding a func. + void *guest_handle = redlopen_guest_library( + dl, handle_snapshot.library); + /* The caller may probe for an optional symbol. */ + if (!guest_handle) { printf_dlsym(LOG_NEVER, "redlopen %p return %p\n", rsymbol, (void*)NULL); return NULL; } - lsassert(ret); - dl->libs[nlib]->x86linkmap = (void *)ret; - ret = RunFunctionWithState( + uintptr_t ret = RunFunctionWithStateInternal( (uintptr_t)my_context->dlprivate->x86dlsym, 2, - dl->libs[nlib]->x86linkmap, symbol); + guest_handle, symbol); printf_dlsym(LOG_DEBUG, "call x86dlsym filename %s is wrapped but not find symbol, dlsym(%p, %s) ret=0x%lx\n", - dl->libs[nlib]->name, dl->libs[nlib]->x86linkmap, (char *)symbol, ret); + handle_snapshot.library->name, + handle_snapshot.library->x86linkmap, + (char *)symbol, ret); return (void *)ret; } #endif lsassert(dl->x86dlsym); - if (dl->libs[nlib]->x86linkmap != handle) { - cpu->regs[R_EDI] = (uintptr_t)dl->libs[nlib]->x86linkmap; + if (handle_snapshot.library->x86linkmap != handle) { + cpu->regs[R_EDI] = + (uintptr_t)handle_snapshot.library->x86linkmap; } #if FORWORDBACK Push64(cpu, (uint64_t)dl->x86dlsym); - printf_dlsym(LOG_DEBUG, "warning call x86dlsym filename is %s %lx\n", dl->libs[nlib]->x86linkmap->l_name, cpu->regs[R_EDI]); + printf_dlsym( + LOG_DEBUG, + "warning call x86dlsym filename is %s %lx\n", + handle_snapshot.library->x86linkmap->l_name, + cpu->regs[R_EDI]); return NULL; #else - uint64_t ret = RunFunctionWithState( + uint64_t ret = RunFunctionWithStateInternal( (uintptr_t)my_context->dlprivate->x86dlsym, 2, - dl->libs[nlib]->x86linkmap, symbol); - printf_dlsym(LOG_DEBUG, "call x86dlsym filename is %s %s ret=0x%lx\n", dl->libs[nlib]->x86linkmap->l_name, (char *)symbol, ret); + handle_snapshot.library->x86linkmap, symbol); + printf_dlsym( + LOG_DEBUG, + "call x86dlsym filename is %s %s ret=0x%lx\n", + handle_snapshot.library->x86linkmap->l_name, + (char *)symbol, ret); + if (ret) { + return (void *)ret; + } + ret = latx_kzt_guest_tls_enabled() + ? kzt_resolve_guest_link_map_symbol( + (uintptr_t)handle_snapshot.library->x86linkmap, + rsymbol) : 0; if (ret) { return (void *)ret; } @@ -617,9 +936,11 @@ EXPORT void* my_dlsym(void *handle, void *symbol){ EXPORT int my_dlclose(void *handle) { + KZT_GUEST_LOADER_OPERATION_GUARD(); printf_dlsym(LOG_DEBUG, "Call to dlclose(%p)\n", handle); dlprivate_t *dl = my_context->dlprivate; CLEARERR + kzt_guest_tls_loader_event_begin(); if (!dl->x86dlclose) { if (init_x86dlfun() != 0 || !dl->x86dlclose) { set_dl_error(dl, "Cannot resolve guest dlfcn entry points"); @@ -640,6 +961,11 @@ EXPORT int my_dlclose(void *handle) if(nlib>=dl->lib_sz) { int ret = -1; if (dl->x86dlclose) { +#ifdef CONFIG_LATX_KZT + if (latx_kzt_guest_tls_enabled()) { + return call_guest_dlclose_with_guest_tls(handle); + } +#endif __MY_CPU; Push64(cpu, (uint64_t)dl->x86dlclose); return 0; @@ -651,7 +977,33 @@ EXPORT int my_dlclose(void *handle) set_dl_errorf(dl, "Bad handle %p (already closed))\n", handle); return -1; } - dl->count[nlib] = dl->count[nlib]-1; +#ifdef CONFIG_LATX_KZT + if (latx_kzt_guest_tls_enabled() && + dl->count[nlib] == 1 && dl->dlopened[nlib] && + dl->libs[nlib] && dl->x86dlclose) { + int idx = GetElfIndex(dl->libs[nlib]); + + if (idx != -1) { + void *guest_handle = dl->libs[nlib]->x86linkmap + ? (void *)dl->libs[nlib]->x86linkmap : handle; + int close_result = + call_guest_dlclose_with_guest_tls(guest_handle); + + if (close_result != 0) { + return close_result; + } + kzt_dl_metadata_lock_acquire(); + dl->count[nlib] = 0; + printf_dlsym( + LOG_DEBUG, "dlclose: Call to Fini for %p\n", handle); + InactiveLibrary(dl->libs[nlib]); + kzt_dl_metadata_lock_release(); + return 0; + } + } +#endif + kzt_dl_metadata_lock_acquire(); + dl->count[nlib] -= 1; if(dl->count[nlib]==0 && dl->dlopened[nlib]) { // need to call Fini... int idx = GetElfIndex(dl->libs[nlib]); if(idx!=-1) { @@ -660,13 +1012,20 @@ EXPORT int my_dlclose(void *handle) if (dl->x86dlclose) { __MY_CPU; if (dl->libs[nlib]->x86linkmap != handle) { - cpu->regs[R_EDI] = (uintptr_t)dl->libs[nlib]->x86linkmap; + cpu->regs[R_EDI] = + (uintptr_t)dl->libs[nlib]->x86linkmap; } Push64(cpu, (uint64_t)dl->x86dlclose); + kzt_dl_metadata_lock_release(); return 0; } } } + kzt_dl_metadata_lock_release(); + if (dl->libs[nlib]) { + kzt_unregister_guest_tls_link_map( + (uintptr_t)dl->libs[nlib]->x86linkmap); + } return 0; } @@ -678,13 +1037,13 @@ EXPORT char* my_dlerror(void) init_x86dlfun(); if (dl_error_pending) { if (dl->x86dlerror) - (void)RunFunctionWithState((uintptr_t)dl->x86dlerror, 0); + (void)RunFunctionWithStateInternal((uintptr_t)dl->x86dlerror, 0); dl_error_pending = 0; return dl_error_buffer; } if (!dl->x86dlerror) return NULL; - return (char*)(uintptr_t)RunFunctionWithState( + return (char *)(uintptr_t)RunFunctionWithStateInternal( (uintptr_t)dl->x86dlerror, 0); } @@ -704,9 +1063,14 @@ EXPORT int my_dladdr1(void *addr, void *i, void** extra_info, int flags) __MY_CPU; uint64_t ret = 0; if (extra_info == NULL && flags == 0) { - ret = RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dladdr, 2, cpu->regs[R_EDI], cpu->regs[R_ESI]); + ret = RunFunctionWithStateInternal( + (uintptr_t)my_context->dlprivate->x86dladdr, 2, + cpu->regs[R_EDI], cpu->regs[R_ESI]); } else { - ret = RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dladdr1, 4, cpu->regs[R_EDI], cpu->regs[R_ESI], cpu->regs[R_EDX], cpu->regs[R_ECX]); + ret = RunFunctionWithStateInternal( + (uintptr_t)my_context->dlprivate->x86dladdr1, 4, + cpu->regs[R_EDI], cpu->regs[R_ESI], + cpu->regs[R_EDX], cpu->regs[R_ECX]); } printf_dlsym(LOG_DEBUG, " call to x86dladdr1 return saddr=%p, fname=\"%s\", sname=\"%s\" ret=%ld\n", info->dli_saddr, info->dli_sname?info->dli_sname:"", info->dli_fname?info->dli_fname:"", ret); if (ret == 1) { @@ -741,7 +1105,9 @@ EXPORT int my_dladdr(void *addr, void *i) #endif printf_dlsym(LOG_DEBUG, "Warning: partially unimplement call to dladdr(%p, %p)\n", addr, info); __MY_CPU; - uint64_t ret = RunFunctionWithState((uintptr_t)my_context->dlprivate->x86dladdr, 2, cpu->regs[R_EDI], cpu->regs[R_ESI]); + uint64_t ret = RunFunctionWithStateInternal( + (uintptr_t)my_context->dlprivate->x86dladdr, 2, + cpu->regs[R_EDI], cpu->regs[R_ESI]); printf_dlsym(LOG_DEBUG, " call to x86dladdr return saddr=%p, fname=\"%s\", sname=\"%s\" ret=%ld\n", info->dli_saddr, info->dli_sname?info->dli_sname:"", info->dli_fname?info->dli_fname:"", ret); if (ret == 1) { return ret; @@ -752,7 +1118,7 @@ EXPORT void* my_dlvsym(void *handle, void *symbol, const char *vername) { printf_dlsym(LOG_DEBUG, "Call to dlvsym(%p, \"%s\", %s)", handle, (char *)symbol, vername?vername:"(nil)"); dlprivate_t *dl = my_context->dlprivate; - size_t nlib; + dl_handle_snapshot_t handle_snapshot = { 0 }; void *guest_handle = handle; clear_dl_error(dl); @@ -768,23 +1134,21 @@ EXPORT void* my_dlvsym(void *handle, void *symbol, const char *vername) return NULL; } if (!handle) - return (void*)(uintptr_t)RunFunctionWithState( + return (void *)(uintptr_t)RunFunctionWithStateInternal( (uintptr_t)dl->x86dlvsym, 3, guest_handle, symbol, vername); - if (find_dl_library_index(dl, handle, &nlib)) { - if (!dl->count[nlib]) { + if (snapshot_dl_handle(dl, handle, &handle_snapshot)) { + if (!handle_snapshot.count) { set_dl_errorf(dl, "Bad handle %p (already closed)\n", handle); return NULL; } - if (!dl->libs[nlib]) { - return (void*)(uintptr_t)RunFunctionWithState( + if (!handle_snapshot.library) { + return (void *)(uintptr_t)RunFunctionWithStateInternal( (uintptr_t)dl->x86dlvsym, 3, NULL, symbol, vername); } - guest_handle = dl->libs[nlib]->x86linkmap; + guest_handle = handle_snapshot.library->x86linkmap; if (!guest_handle) { - guest_handle = (void*)(uintptr_t)RunFunctionWithState( - (uintptr_t)dl->x86dlopen, 2, dl->libs[nlib]->name, - dl->libs[nlib]->x86dlopenflag); - dl->libs[nlib]->x86linkmap = guest_handle; + guest_handle = redlopen_guest_library( + dl, handle_snapshot.library); if (!guest_handle) { set_dl_errorf(dl, "Missing guest link_map for handle %p\n", handle); @@ -792,7 +1156,7 @@ EXPORT void* my_dlvsym(void *handle, void *symbol, const char *vername) } } } - uintptr_t ret = RunFunctionWithState( + uintptr_t ret = RunFunctionWithStateInternal( (uintptr_t)dl->x86dlvsym, 3, guest_handle, symbol, vername); return (void*)ret; } @@ -808,22 +1172,20 @@ EXPORT int my_dlinfo(void* handle, int request, void* info) return -1; } } - size_t nlib; + dl_handle_snapshot_t handle_snapshot = { 0 }; void *guest_handle = handle; - if (find_dl_library_index(dl, handle, &nlib)) { - if (!dl->count[nlib]) { + if (snapshot_dl_handle(dl, handle, &handle_snapshot)) { + if (!handle_snapshot.count) { set_dl_errorf(dl, "Bad handle %p (already closed)\n", handle); return -1; } - if (!dl->libs[nlib]) { + if (!handle_snapshot.library) { guest_handle = NULL; } else { - guest_handle = dl->libs[nlib]->x86linkmap; + guest_handle = handle_snapshot.library->x86linkmap; if (!guest_handle) { - guest_handle = (void*)(uintptr_t)RunFunctionWithState( - (uintptr_t)dl->x86dlopen, 2, dl->libs[nlib]->name, - dl->libs[nlib]->x86dlopenflag); - dl->libs[nlib]->x86linkmap = guest_handle; + guest_handle = redlopen_guest_library( + dl, handle_snapshot.library); if (!guest_handle) { set_dl_errorf(dl, "Cannot open guest library for handle %p\n", @@ -843,12 +1205,13 @@ EXPORT int my_dlinfo(void* handle, int request, void* info) } } } - uint64_t ret = RunFunctionWithState( + uint64_t ret = RunFunctionWithStateInternal( (uintptr_t)my_context->dlprivate->x86dlinfo, 3, guest_handle, request, info); return ret; } + #ifndef CONFIG_LOONGARCH_NEW_WORLD #include "wrappedlib_init.h" #endif diff --git a/target/i386/latx/context/wrappedlibx11.c b/target/i386/latx/context/wrappedlibx11.c index a729112dcb..7923492a75 100644 --- a/target/i386/latx/context/wrappedlibx11.c +++ b/target/i386/latx/context/wrappedlibx11.c @@ -1266,7 +1266,7 @@ EXPORT int32_t my_XDestroyImage(void* image) void *la_data = malloc(len); memcpy(la_data, img->data, len); lsassert(x86free); - RunFunctionWithState((uintptr_t)x86free ,1, img->data); + RunFunctionWithStateInternal((uintptr_t)x86free, 1, img->data); img->data = la_data; } return my->XDestroyImage(image); @@ -1653,9 +1653,12 @@ EXPORT int32_t my_XNextEvent(my_XDisplay_t *dpy, void* v2) int oldtype; int32_t ret; bridge_XInternalAsyncHandlers(dpy, bridge_XInternalAsyncHandler); - uint64_t callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype ,2, PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); + uint64_t callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, + PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); ret = my->XNextEvent(dpy,v2); - callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype ,2, oldtype, NULL); + callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, oldtype, NULL); (void)callbackret; return ret; } diff --git a/target/i386/latx/context/wrappedlibxcb.c b/target/i386/latx/context/wrappedlibxcb.c index 016d91515a..6dc1cc3f8b 100644 --- a/target/i386/latx/context/wrappedlibxcb.c +++ b/target/i386/latx/context/wrappedlibxcb.c @@ -36,9 +36,12 @@ EXPORT void* my_xcb_wait_for_event(void* v1) { int oldtype; void* ret; - uint64_t callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype , 2, PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); + uint64_t callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, + PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); ret = my->xcb_wait_for_event(v1); - callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype , 2, oldtype, NULL); + callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, oldtype, NULL); (void)callbackret; return ret; } @@ -55,9 +58,12 @@ EXPORT void* my_xcb_wait_for_reply(void* v1, uint32_t v2, void* v3) { int oldtype; void* ret; - uint64_t callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype , 2, PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); + uint64_t callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, + PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); ret = my->xcb_wait_for_reply(v1, v2, v3); - callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype , 2, oldtype, NULL); + callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, oldtype, NULL); (void)callbackret; return ret; } @@ -67,9 +73,12 @@ EXPORT void* my_xcb_wait_for_reply64(void* v1, uint64_t v2, void* v3) { int oldtype; void* ret; - uint64_t callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype , 2, PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); + uint64_t callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, + PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); ret = my->xcb_wait_for_reply64(v1, v2, v3); - callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype , 2, oldtype, NULL); + callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, oldtype, NULL); (void)callbackret; return ret; } @@ -79,9 +88,12 @@ EXPORT void* my_xcb_wait_for_special_event(void* v1, void* v2) { int oldtype; void* ret; - uint64_t callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype , 2, PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); + uint64_t callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, + PTHREAD_CANCEL_ASYNCHRONOUS, &oldtype); ret = my->xcb_wait_for_special_event(v1, v2); - callbackret = RunFunctionWithState((uintptr_t)x86pthread_setcanceltype , 2, oldtype, NULL); + callbackret = RunFunctionWithStateInternal( + (uintptr_t)x86pthread_setcanceltype, 2, oldtype, NULL); (void)callbackret; return ret; } diff --git a/target/i386/latx/context/x86dlfun.c b/target/i386/latx/context/x86dlfun.c index 5eb4761040..b991f16b73 100644 --- a/target/i386/latx/context/x86dlfun.c +++ b/target/i386/latx/context/x86dlfun.c @@ -12,6 +12,7 @@ #include "elfloader.h" #include "latx-options.h" #include "myalign.h" +#include "kzt-guest-tls.h" #include "x86dlfun.h" static void set_x86dlfun(void *const *resolved) @@ -37,6 +38,21 @@ int init_x86dlfun_from(const char *primary, const char *fallback) elfheader_t *header; int resolved_count = 0; + if (latx_kzt_guest_tls_enabled()) { + for (int index = 0; index < X86_DL_SYMBOL_COUNT; ++index) { + resolved[index] = (void *)kzt_resolve_guest_object_symbol( + primary, symbols[index]); + if (!resolved[index]) { + resolved[index] = (void *)kzt_resolve_guest_object_symbol( + fallback, symbols[index]); + } + if (!resolved[index]) { + return -1; + } + } + set_x86dlfun(resolved); + return 0; + } #if defined(CONFIG_LOONGARCH_NEW_WORLD) && defined(CONFIG_LATX_KZT) if (latx_kzt_runtime_enabled()) { for (int index = 0; index < X86_DL_SYMBOL_COUNT; ++index) { diff --git a/target/i386/latx/include/callback.h b/target/i386/latx/include/callback.h index cf1b5a9daf..e64fed103a 100644 --- a/target/i386/latx/include/callback.h +++ b/target/i386/latx/include/callback.h @@ -4,10 +4,22 @@ #include uint64_t RunFunctionWithState(uintptr_t fnc, int nargs, ...); +uint64_t RunFunctionWithStateInternal(uintptr_t fnc, int nargs, ...); +uint64_t RunFunctionWithStateInternalNoRefresh(uintptr_t fnc, int nargs, + ...); uint64_t RunFunctionFmt(uintptr_t fnc, const char *fmt, ...); float RunFunctionFmtFloat(uintptr_t fnc, const char *fmt, ...); -uint64_t RunFunctionWithStateInternal(uintptr_t fnc, int nargs, ...); -uint64_t RunFunctionWithStateInternalNoRefresh(uintptr_t fnc, int nargs, ...); #define RunFunction RunFunctionWithState +int latx_guest_internal_callback_active(void); + +int latx_run_guest_callback(uintptr_t entry, const long *gpr_args, + int gpr_count, const long *xmm_args, + int xmm_count, const long *stack_args, + int stack_count, long *rax, long *rdx, + long *xmm0, long *xmm1, + unsigned __int128 *st0); + + + #endif //__CALLBACK_H__ diff --git a/target/i386/latx/include/elfloader.h b/target/i386/latx/include/elfloader.h index 2c51770d5f..9cd5474ba1 100755 --- a/target/i386/latx/include/elfloader.h +++ b/target/i386/latx/include/elfloader.h @@ -55,6 +55,7 @@ uint32_t GetBaseSize(elfheader_t* h); int IsAddressInElfSpace(const elfheader_t* h, uintptr_t addr); elfheader_t* FindElfAddress(box64context_t *context, uintptr_t addr); const char* FindNearestSymbolName(elfheader_t* h, void* p, uintptr_t* start, uint64_t* sz); +uintptr_t FindElfSymbolAddress(elfheader_t *h, const char *name); void* GetDynamicSection(elfheader_t* h); const char* GetSymbolVersion(elfheader_t* h, int version); diff --git a/target/i386/latx/include/kzt-guest-thread.h b/target/i386/latx/include/kzt-guest-thread.h new file mode 100644 index 0000000000..80b3cb35c2 --- /dev/null +++ b/target/i386/latx/include/kzt-guest-thread.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef LATX_KZT_GUEST_THREAD_H +#define LATX_KZT_GUEST_THREAD_H + +#include + +typedef struct CPUX86State CPUX86State; + +#ifdef CONFIG_LATX_KZT +void kzt_guest_thread_initialize(CPUX86State *env); +void kzt_guest_thread_destroy(CPUX86State *env); +void kzt_guest_thread_after_fork_child(void); +int kzt_guest_thread_key_create(CPUX86State *env, unsigned int *key, + uintptr_t destructor); +int kzt_guest_thread_key_delete(CPUX86State *env, unsigned int key); +int kzt_guest_thread_cxa_atexit(CPUX86State *env, uintptr_t destructor, + uintptr_t object, uintptr_t dso_handle); +#else +static inline void kzt_guest_thread_initialize(CPUX86State *env) {} +static inline void kzt_guest_thread_destroy(CPUX86State *env) {} +static inline void kzt_guest_thread_after_fork_child(void) {} +static inline int kzt_guest_thread_key_create(CPUX86State *env, + unsigned int *key, uintptr_t destructor) { return -1; } +static inline int kzt_guest_thread_key_delete(CPUX86State *env, + unsigned int key) { return -1; } +static inline int kzt_guest_thread_cxa_atexit(CPUX86State *env, + uintptr_t destructor, uintptr_t object, uintptr_t dso_handle) { return -1; } +#endif + +#endif diff --git a/target/i386/latx/include/kzt-guest-tls-epoch.h b/target/i386/latx/include/kzt-guest-tls-epoch.h new file mode 100644 index 0000000000..49e925dfc5 --- /dev/null +++ b/target/i386/latx/include/kzt-guest-tls-epoch.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef LATX_KZT_GUEST_TLS_EPOCH_H +#define LATX_KZT_GUEST_TLS_EPOCH_H + +#include + +static inline uint32_t kzt_guest_tls_epoch_begin_change( + uint32_t sequence) +{ + if (!sequence || (sequence & 1)) { + return sequence; + } + return sequence + 1; +} + +static inline uint32_t kzt_guest_tls_epoch_publish_stable( + uint32_t sequence) +{ + if (!(sequence & 1) || sequence == UINT32_MAX) { + return sequence; + } + return sequence + 1; +} + +static inline uint32_t kzt_guest_tls_epoch_after_fork( + uint32_t sequence) +{ + return sequence ? 1 : 0; +} + +static inline int kzt_guest_tls_epoch_can_reuse( + uint32_t sequence_before, + uint32_t sequence_after, + int loader_consistent, + uintptr_t pending_generation, + uintptr_t state_generation, + uintptr_t process_generation) +{ + return sequence_before != 0 && + !(sequence_before & 1) && + sequence_before == sequence_after && + loader_consistent && + pending_generation == 0 && + state_generation == process_generation; +} + +#endif diff --git a/target/i386/latx/include/kzt-guest-tls.h b/target/i386/latx/include/kzt-guest-tls.h new file mode 100644 index 0000000000..a68f0e675f --- /dev/null +++ b/target/i386/latx/include/kzt-guest-tls.h @@ -0,0 +1,203 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef LATX_KZT_GUEST_TLS_H +#define LATX_KZT_GUEST_TLS_H + +#include +#include + +#ifdef CONFIG_LATX_KZT +#include "kzt-runtime.h" +#else +static inline bool latx_kzt_guest_tls_enabled(void) +{ + return false; +} +#endif + +typedef struct CPUX86State CPUX86State; + +#define KZT_GUEST_TLS_REFRESH_BUSY 1 + +#ifdef CONFIG_LATX_KZT +int kzt_guest_tls_snapshot_parent(CPUX86State *parent, + CPUX86State *child); +int kzt_guest_tls_clone_parent_snapshot(const CPUX86State *parent, + CPUX86State *child); +/* BUSY is returned only before allocating or executing Guest TLS state. */ +int kzt_guest_tls_initialize(CPUX86State *parent, CPUX86State *child); +int kzt_guest_tls_refresh(CPUX86State *env); +int kzt_guest_tls_refresh_if_needed(CPUX86State *env); +int kzt_guest_tls_refresh_local(CPUX86State *env); +void kzt_guest_tls_loader_tracking_enable(void); +void kzt_guest_tls_loader_tracking_reset(void); +void kzt_guest_tls_loader_event_begin(void); +int kzt_guest_tls_loader_event_observe(void); +int kzt_guest_loader_operation_active(void); +void kzt_guest_tls_execution_enter(CPUX86State *env); +void kzt_guest_tls_execution_leave(CPUX86State *env); +int kzt_guest_tls_execution_pause(CPUX86State *env); +void kzt_guest_tls_execution_resume(CPUX86State *env, int paused); +void kzt_guest_tls_fork_prepare(void); +void kzt_guest_tls_fork_prepare_early(void); +void kzt_guest_tls_fork_parent(void); +void kzt_guest_tls_fork_abort(void); +int kzt_guest_tls_syscall_pause(CPUX86State *env); +void kzt_guest_tls_after_fork_child(CPUX86State *env); +void kzt_guest_loader_after_fork_child(void); +uintptr_t kzt_guest_loader_hold_open(uintptr_t guest_name); +void kzt_guest_loader_hold_close(uintptr_t handle); +int kzt_guest_tls_preinitialize_static(CPUX86State *env, + uintptr_t link_map_addr); +void kzt_guest_tls_destroy(CPUX86State *env); +int kzt_guest_tls_cleanup_robust_list(CPUX86State *env); +#else +static inline int kzt_guest_tls_snapshot_parent(CPUX86State *parent, + CPUX86State *child) +{ + (void)parent; + (void)child; + return 0; +} + +static inline int kzt_guest_tls_clone_parent_snapshot( + const CPUX86State *parent, CPUX86State *child) +{ + (void)parent; + (void)child; + return 0; +} + +static inline int kzt_guest_tls_initialize(CPUX86State *parent, + CPUX86State *child) +{ + (void)parent; + (void)child; + return 0; +} + +static inline void kzt_guest_tls_destroy(CPUX86State *env) +{ + (void)env; +} + +static inline int kzt_guest_tls_cleanup_robust_list(CPUX86State *env) +{ + (void)env; + return 0; +} + +static inline int kzt_guest_tls_refresh(CPUX86State *env) +{ + (void)env; + return 0; +} + +static inline int kzt_guest_tls_refresh_if_needed(CPUX86State *env) +{ + (void)env; + return 0; +} + +static inline int kzt_guest_tls_refresh_local(CPUX86State *env) +{ + (void)env; + return 0; +} + +static inline void kzt_guest_tls_loader_tracking_enable(void) +{ +} + +static inline void kzt_guest_tls_loader_tracking_reset(void) +{ +} + +static inline void kzt_guest_tls_loader_event_begin(void) +{ +} + +static inline int kzt_guest_tls_loader_event_observe(void) +{ + return 0; +} + +static inline int kzt_guest_loader_operation_active(void) +{ + return 0; +} + +static inline void kzt_guest_tls_execution_enter(CPUX86State *env) +{ + (void)env; +} + +static inline void kzt_guest_tls_execution_leave(CPUX86State *env) +{ + (void)env; +} + +static inline int kzt_guest_tls_execution_pause(CPUX86State *env) +{ + (void)env; + return 0; +} + +static inline void kzt_guest_tls_execution_resume(CPUX86State *env, + int paused) +{ + (void)env; + (void)paused; +} + +static inline void kzt_guest_tls_fork_prepare(void) +{ +} + +static inline void kzt_guest_tls_fork_prepare_early(void) +{ +} + +static inline void kzt_guest_tls_fork_abort(void) {} +static inline int kzt_guest_tls_syscall_pause(CPUX86State *env) +{ + (void)env; + return 0; +} + +static inline void kzt_guest_tls_fork_parent(void) +{ +} + +static inline void kzt_guest_tls_after_fork_child(CPUX86State *env) +{ + (void)env; +} + +static inline void kzt_guest_loader_after_fork_child(void) +{ +} + +static inline uintptr_t kzt_guest_loader_hold_open( + uintptr_t guest_name) +{ + (void)guest_name; + return 0; +} + +static inline void kzt_guest_loader_hold_close(uintptr_t handle) +{ + (void)handle; +} + +static inline int kzt_guest_tls_preinitialize_static( + CPUX86State *env, uintptr_t link_map_addr) +{ + (void)env; + (void)link_map_addr; + return 0; +} + +#endif + +#endif diff --git a/target/i386/latx/include/kzt-runtime.h b/target/i386/latx/include/kzt-runtime.h index 906cb7fd55..fa5a4986f7 100644 --- a/target/i386/latx/include/kzt-runtime.h +++ b/target/i386/latx/include/kzt-runtime.h @@ -21,6 +21,7 @@ * cached "active" flag. */ extern int option_kzt; +extern int option_kzt_guest_tls; extern uint32_t kzt_effective_groups; static inline bool latx_kzt_runtime_enabled(void) @@ -28,4 +29,10 @@ static inline bool latx_kzt_runtime_enabled(void) return option_kzt != 0 && kzt_effective_groups != 0; } +/* Resolved at startup; never toggle ownership after threads are attached. */ +static inline bool latx_kzt_guest_tls_enabled(void) +{ + return option_kzt_guest_tls != 0 && latx_kzt_runtime_enabled(); +} + #endif /* LATX_KZT_RUNTIME_H */ diff --git a/target/i386/latx/include/kzt_public_loader_observer.h b/target/i386/latx/include/kzt_public_loader_observer.h index b00ba55507..05f8a50412 100644 --- a/target/i386/latx/include/kzt_public_loader_observer.h +++ b/target/i386/latx/include/kzt_public_loader_observer.h @@ -64,6 +64,26 @@ typedef struct kzt_public_loader_object { uintptr_t previous_addr; } kzt_public_loader_object_t; +typedef struct kzt_public_loader_tls_object { + uintptr_t link_map_addr; + uintptr_t load_bias; + uintptr_t dynamic_addr; + uintptr_t image_addr; + size_t file_size; + size_t memory_size; + size_t alignment; + size_t first_byte_offset; + intptr_t static_tls_offset; + uintptr_t static_tls_symbol_value; + uintptr_t static_tls_symbol_name_addr; + size_t module_id; + uint64_t load_generation; + int external_registration; + int static_tls_offset_valid; + int static_tls_offset_needs_validation; + int static_tls_offset_pending; +} kzt_public_loader_tls_object_t; + typedef int (*kzt_public_loader_visit_fn)( const kzt_public_loader_object_t *object, void *opaque); @@ -85,7 +105,9 @@ typedef struct kzt_public_loader_observer { uintptr_t r_debug_addr; uintptr_t r_brk_addr; uintptr_t live_maps[KZT_PUBLIC_LOADER_MAX_OBJECTS]; + uint64_t live_map_generations[KZT_PUBLIC_LOADER_MAX_OBJECTS]; size_t live_map_count; + uint64_t next_load_generation; uintptr_t processed_maps[KZT_PUBLIC_LOADER_MAX_OBJECTS]; size_t processed_map_count; uintptr_t fallback_reported_maps[KZT_PUBLIC_LOADER_MAX_OBJECTS]; @@ -178,6 +200,11 @@ kzt_public_loader_result_t kzt_public_loader_observer_refresh( kzt_public_loader_visit_fn visit, void *visit_opaque); +/* Probe the current public loader state without walking the link_map chain. */ +kzt_public_loader_result_t kzt_public_loader_state_is_consistent( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader); + /* Resolve one unique defined symbol from the live in-memory link_map set. */ kzt_public_loader_result_t kzt_public_loader_find_symbol( const kzt_public_loader_observer_t *observer, @@ -185,6 +212,61 @@ kzt_public_loader_result_t kzt_public_loader_find_symbol( const char *symbol_name, uintptr_t *symbol_addr); +kzt_public_loader_result_t kzt_public_loader_find_symbol_in_object( + const kzt_public_loader_object_t *object, + const kzt_public_loader_reader_t *reader, + const char *symbol_name, + uintptr_t *symbol_addr); + +/* + * Resolve one observed object whose exact, unaligned PT_LOAD memory range + * contains guest_addr. A link_map already remembered by the pre-protection + * path may be resolved during RT_ADD; an address not found during RT_ADD stays + * BUSY until the loader publishes a complete snapshot. RT_DELETE is rejected. + * PT_LOAD file-to-BSS gaps are included through p_memsz, while inter-segment + * gaps and segment end addresses are excluded. + */ +kzt_public_loader_result_t kzt_public_loader_find_object_by_address( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader, + uintptr_t guest_addr, + kzt_public_loader_object_t *object); + +kzt_public_loader_result_t kzt_public_loader_object_contains_address( + const kzt_public_loader_object_t *object, + const kzt_public_loader_reader_t *reader, + uintptr_t guest_addr, + int *contains); + +kzt_public_loader_result_t kzt_public_loader_collect_tls( + const kzt_public_loader_observer_t *observer, + const kzt_public_loader_reader_t *reader, + kzt_public_loader_tls_object_t *objects, + size_t object_capacity, + size_t *object_count); + +kzt_public_loader_result_t kzt_public_loader_read_tls_object( + const kzt_public_loader_object_t *object, + const kzt_public_loader_reader_t *reader, + kzt_public_loader_tls_object_t *tls_object, + int *has_tls); +kzt_public_loader_result_t kzt_public_loader_materialize_tls_image( + const kzt_public_loader_tls_object_t *object, + const kzt_public_loader_reader_t *reader, + void *destination, + size_t destination_size); + +/* Refresh a private observer copy and collect TLS without committing state. */ +kzt_public_loader_result_t kzt_public_loader_snapshot_tls( + const kzt_public_loader_observer_t *observer, + uintptr_t dynamic_addr, + size_t max_dynamic_entries, + const kzt_public_loader_reader_t *reader, + uintptr_t link_map_filter, + kzt_public_loader_tls_object_t *objects, + size_t object_capacity, + size_t *object_count); + const char *kzt_public_loader_result_name( kzt_public_loader_result_t result); diff --git a/target/i386/latx/include/latx-options.h b/target/i386/latx/include/latx-options.h index cf8e13453f..6f61ca8277 100644 --- a/target/i386/latx/include/latx-options.h +++ b/target/i386/latx/include/latx-options.h @@ -37,6 +37,7 @@ extern int option_tu_link; extern int option_kzt_log; extern char *option_kzt_libs; extern char *option_kzt_error; +extern char *option_kzt_guest_tls_error; extern char *option_kzt_log_error; #endif @@ -188,6 +189,7 @@ extern unsigned long long counter_mips_tr; #if defined(CONFIG_LATX) && defined(CONFIG_LATX_KZT) #define ENVSUP_KZT \ ENVFUN(LATX_KZT, handle_arg_latx_kzt) \ + ENVFUN(LATX_KZT_GUEST_TLS, handle_arg_latx_kzt_guest_tls) \ ENVFUN(LATX_KZT_LIBS, handle_arg_latx_kzt_libs) \ ENVFUN(LATX_KZT_LOG, handle_arg_latx_kzt_log) #else diff --git a/target/i386/latx/include/myalign.h b/target/i386/latx/include/myalign.h index 8d94c5a231..16647bd7ac 100644 --- a/target/i386/latx/include/myalign.h +++ b/target/i386/latx/include/myalign.h @@ -17,6 +17,7 @@ #include "translate.h" #include "wrappertbbridge.h" #include "callback.h" +#include "kzt_public_loader_observer.h" extern elfheader_t* elf_header; extern int latx_wine; @@ -160,5 +161,30 @@ void kzt_bridge_init(void); void kzt_wine_bridge(abi_ulong start, int fd); int latx_dpy_xcb_sync(void *v1); uintptr_t kzt_resolve_guest_symbol(const char *name); +uintptr_t kzt_resolve_guest_object_symbol( + const char *object_name, + const char *symbol_name); +uintptr_t kzt_resolve_guest_link_map_symbol( + uintptr_t link_map_addr, + const char *symbol_name); +uintptr_t kzt_find_guest_link_map_by_address(uintptr_t guest_addr); +int kzt_guest_loader_state_is_consistent(void); +uintptr_t kzt_find_guest_link_map_by_address_ex( + uintptr_t guest_addr, + kzt_public_loader_result_t *lookup_result); +int kzt_register_guest_tls_link_map(uintptr_t link_map_addr); +void kzt_unregister_guest_tls_link_map(uintptr_t link_map_addr); +int kzt_collect_guest_tls_objects( + kzt_public_loader_tls_object_t *objects, + size_t object_capacity, + size_t *object_count); +int kzt_collect_guest_tls_object( + uintptr_t link_map_addr, + kzt_public_loader_tls_object_t *object, + int *has_tls); +int kzt_materialize_guest_tls_image( + const kzt_public_loader_tls_object_t *object, + void *destination, + size_t destination_size); elfheader_t* loadElfFromFile(const char* name); #endif //__MY_ALIGN__H_ diff --git a/target/i386/latx/include/wrappedlibc_private.h b/target/i386/latx/include/wrappedlibc_private.h index f836a95f9b..32f919a638 100644 --- a/target/i386/latx/include/wrappedlibc_private.h +++ b/target/i386/latx/include/wrappedlibc_private.h @@ -188,7 +188,9 @@ //GOM(__cxa_atexit, iFEppp) ////GO(__cxa_at_quick_exit, /* at_quick_exit has signature iF@ -> */ iF@pp) //GOM(__cxa_finalize, vFEp) -//GOM(__cxa_thread_atexit_impl, iFEppp) +#ifdef CONFIG_LATX_KZT +GOM(__cxa_thread_atexit_impl, iFEppp) +#endif ////GO(__cyg_profile_func_enter, ////GO(__cyg_profile_func_exit, //GO(daemon, iFii) @@ -1467,8 +1469,11 @@ GOM(__libc_free, vFp) //GO(pthread_getspecific, pFL) //GO(pthread_getname_np, iFppL) //GO(pthread_join, iFLp) -//GOM(pthread_key_create, iFEpp) -//GO(pthread_key_delete, iFL) +#ifdef CONFIG_LATX_KZT +GOM(pthread_key_create, iFEpp) +GOM(__pthread_key_create, iFEpp) +GOM(pthread_key_delete, iFu) +#endif //GO2(pthread_kill@GLIBC_2.2.5, iFEpi, my_pthread_kill_old) //GOM(pthread_kill, iFEpi) //GO(pthread_kill_other_threads_np, vFv) diff --git a/target/i386/latx/latx-options.c b/target/i386/latx/latx-options.c index cd60cd3e90..cb1e4128d1 100644 --- a/target/i386/latx/latx-options.c +++ b/target/i386/latx/latx-options.c @@ -18,6 +18,8 @@ #if defined(CONFIG_LATX_KZT) int option_kzt = 0; +int option_kzt_guest_tls = 0; +char *option_kzt_guest_tls_error; int option_kzt_log = 0; char *option_kzt_libs; char *option_kzt_error; @@ -221,6 +223,8 @@ void options_init(void) latx_runtime_reset(); #if defined(CONFIG_LATX_KZT) option_kzt = 0; + option_kzt_guest_tls = 0; + g_clear_pointer(&option_kzt_guest_tls_error, g_free); option_kzt_log = 0; g_clear_pointer(&option_kzt_libs, g_free); g_clear_pointer(&option_kzt_error, g_free); @@ -294,6 +298,11 @@ void options_init(void) bool latx_options_finalize(void) { #if defined(CONFIG_LATX_KZT) + if (option_kzt_guest_tls_error) { + kzt_groups_reject_configuration(option_kzt_guest_tls_error, true); + option_kzt = 0; + return false; + } if (option_kzt_log_error) { kzt_groups_reject_configuration(option_kzt_log_error, true); option_kzt = 0; diff --git a/target/i386/latx/sbt/tests/aot-cache-reader-test.c b/target/i386/latx/sbt/tests/aot-cache-reader-test.c index 8048631fe4..56a07a1c06 100644 --- a/target/i386/latx/sbt/tests/aot-cache-reader-test.c +++ b/target/i386/latx/sbt/tests/aot-cache-reader-test.c @@ -1,4 +1,5 @@ #include "qemu/osdep.h" +#include #include diff --git a/target/i386/latx/sbt/tests/aot-file-publish-test.c b/target/i386/latx/sbt/tests/aot-file-publish-test.c index b304e49f51..50108c2ce5 100644 --- a/target/i386/latx/sbt/tests/aot-file-publish-test.c +++ b/target/i386/latx/sbt/tests/aot-file-publish-test.c @@ -1,4 +1,5 @@ #include "qemu/osdep.h" +#include #include "file_ctx.h" #include "aot-file-publish-test.h" diff --git a/target/i386/latx/sbt/tests/aot-merge-memory-test.c b/target/i386/latx/sbt/tests/aot-merge-memory-test.c index f02fc43f2a..03f602260a 100644 --- a/target/i386/latx/sbt/tests/aot-merge-memory-test.c +++ b/target/i386/latx/sbt/tests/aot-merge-memory-test.c @@ -5,6 +5,7 @@ */ #include "qemu-def.h" +#include #ifdef AOT_MERGE_TEST_NO_TU #undef CONFIG_LATX_TU diff --git a/target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c b/target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c index 82399ac549..5d29612bf7 100644 --- a/target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c +++ b/target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c @@ -27,6 +27,8 @@ int qemu_loglevel; __thread int in_pre_translate; #if defined(CONFIG_LATX_KZT) int option_kzt; +int option_kzt_guest_tls; +uint32_t kzt_effective_groups; struct image_info info1; #endif diff --git a/tests/integration/kzt-attach-busy.gdb b/tests/integration/kzt-attach-busy.gdb new file mode 100644 index 0000000000..d28d14a48c --- /dev/null +++ b/tests/integration/kzt-attach-busy.gdb @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Requires a LATX debug-symbol build and the native-thread opt-in fixture. +set pagination off +set confirm off +set print thread-events off +set $injected = 0 +set $mode = 0 +break kzt_collect_guest_tls_objects if lsenv != 0 \ + && ($mode == 2 || $injected == 0) \ + && ((CPUX86State *)lsenv->cpu_state)->kzt_guest_tls_parent_snapshot != 0 \ + && ((CPUX86State *)lsenv->cpu_state)->kzt_guest_tls_allocation == 0 +commands + silent + set $injected = $injected + 1 + if $mode == 1 + return (int)-1 + else + return (int)1 + end + continue +end + +# One transient busy result must not drop a valid callback. +run +if $injected != 1 || $_exitcode != 0 + echo FAIL: attached callback did not recover from transient loader busy\n + quit 1 +end +echo PASS: attached callback retried transient loader busy\n + +# A permanent error must not be retried into apparent success. +set $injected = 0 +set $mode = 1 +run +if $injected != 1 || $_exitcode == 0 + echo FAIL: permanent attachment failure was ignored\n + quit 1 +end +echo PASS: permanent attachment failure remains an error\n + +# The fixture starts two native threads. Each has a bounded retry budget. +set $injected = 0 +set $mode = 2 +run +if $injected < 2 || $injected > 64 || $_exitcode == 0 + echo FAIL: persistent loader busy was ignored or exceeded the retry budget\n + quit 1 +end +echo PASS: persistent loader busy is bounded\n diff --git a/tests/integration/kzt-attached-robust-guest.c b/tests/integration/kzt-attached-robust-guest.c new file mode 100644 index 0000000000..e77894fc57 --- /dev/null +++ b/tests/integration/kzt-attached-robust-guest.c @@ -0,0 +1,130 @@ +#define _GNU_SOURCE +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static pthread_mutex_t *mutex; +static int waiter_result; + +static uint32_t read_mutex_word(void) +{ + return __atomic_load_n( + (uint32_t *)&mutex->__data.__lock, __ATOMIC_RELAXED); +} + +static void timeout_handler(int signal_number) +{ + (void)signal_number; + dprintf(STDERR_FILENO, "TIMEOUT word=0x%08x\n", + read_mutex_word()); + _exit(124); +} + +static Bool attached_owner(Display *display, xReply *reply, + char *buffer, int length, XPointer opaque) +{ + long tid = syscall(SYS_gettid); + int result; + + (void)reply; + (void)buffer; + (void)length; + (void)opaque; + result = pthread_mutex_lock(mutex); + fprintf(stderr, + "ATTACHED_LOCK_RESULT=%d tid=%ld word=0x%08x owner=%d\n", + result, tid, read_mutex_word(), + mutex->__data.__owner); + if (result != 0 || + (read_mutex_word() & FUTEX_TID_MASK) != + (uint32_t)tid || mutex->__data.__owner != tid) { + return True; + } + return XNoOp(display) != 0; +} + +static void *run_waiter(void *opaque) +{ + (void)opaque; + waiter_result = pthread_mutex_lock(mutex); + if (waiter_result == EOWNERDEAD) { + pthread_mutex_consistent(mutex); + pthread_mutex_unlock(mutex); + } + return NULL; +} + +int main(void) +{ + const struct timespec poll_delay = { + .tv_nsec = 1000 * 1000, + }; + pthread_mutexattr_t attr; + pthread_t waiter; + Display *display; + _XAsyncHandler handler = { 0 }; + struct sigaction alarm_action = { + .sa_handler = timeout_handler, + }; + uint32_t lock_word = 0; + + mutex = mmap(NULL, sizeof(*mutex), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + display = calloc(1, sizeof(*display)); + if (mutex == MAP_FAILED || !display || + pthread_mutexattr_init(&attr) != 0 || + pthread_mutexattr_setpshared( + &attr, PTHREAD_PROCESS_SHARED) != 0 || + pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) != 0 || + pthread_mutex_init(mutex, &attr) != 0) { + return 2; + } + pthread_mutexattr_destroy(&attr); + handler.handler = attached_owner; + display->async_handlers = &handler; + (void)dlerror(); + if (XEventsQueued(display, 0) != 77 || + pthread_create(&waiter, NULL, run_waiter, NULL) != 0) { + return 3; + } + for (int attempt = 0; attempt < 2000; ++attempt) { + lock_word = read_mutex_word(); + if (lock_word & FUTEX_WAITERS) { + break; + } + nanosleep(&poll_delay, NULL); + } + if (!(lock_word & FUTEX_WAITERS)) { + return 4; + } + fprintf(stderr, "WAITER_BLOCKED word=0x%08x\n", lock_word); + sigemptyset(&alarm_action.sa_mask); + if (sigaction(SIGALRM, &alarm_action, NULL) != 0) { + return 7; + } + alarm(2); + if (XFlush(display) != 78 || pthread_join(waiter, NULL) != 0) { + return 5; + } + alarm(0); + fprintf(stderr, "WAITER_RESULT=%d word=0x%08x\n", + waiter_result, read_mutex_word()); + if (waiter_result != EOWNERDEAD) { + return 6; + } + puts("PASS: attached robust owner death woke blocked Guest waiter"); + return 0; +} diff --git a/tests/integration/kzt-attached-robust-probe.c b/tests/integration/kzt-attached-robust-probe.c new file mode 100644 index 0000000000..be9aa92a4f --- /dev/null +++ b/tests/integration/kzt-attached-robust-probe.c @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include + +#include + +typedef struct owner_call { + Display *display; + int result; +} owner_call; + +static pthread_mutex_t state_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t state_cond = PTHREAD_COND_INITIALIZER; +static pthread_t owner_thread; +static owner_call call; +static int owner_started; +static int owner_ready; +static int owner_finished; +static int release_owner; + +static void *run_owner(void *opaque) +{ + owner_call *owner = opaque; + xReply reply; + + memset(&reply, 0, sizeof(reply)); + owner->result = owner->display->async_handlers->handler( + owner->display, &reply, (char *)&reply, + sizeof(reply), owner->display->async_handlers->data); + pthread_mutex_lock(&state_lock); + owner_finished = 1; + pthread_cond_broadcast(&state_cond); + pthread_mutex_unlock(&state_lock); + return NULL; +} + +int XNoOp(Display *display) +{ + (void)display; + pthread_mutex_lock(&state_lock); + owner_ready = 1; + pthread_cond_broadcast(&state_cond); + while (!release_owner) { + pthread_cond_wait(&state_cond, &state_lock); + } + pthread_mutex_unlock(&state_lock); + return 0; +} + +int XEventsQueued(Display *display, int mode) +{ + (void)mode; + if (!display || !display->async_handlers) { + return -1; + } + pthread_mutex_lock(&state_lock); + call.display = display; + call.result = -1; + owner_ready = 0; + owner_finished = 0; + release_owner = 0; + if (pthread_create(&owner_thread, NULL, run_owner, &call) != 0) { + pthread_mutex_unlock(&state_lock); + return -1; + } + owner_started = 1; + while (!owner_ready && !owner_finished) { + pthread_cond_wait(&state_cond, &state_lock); + } + if (owner_finished) { + pthread_mutex_unlock(&state_lock); + pthread_join(owner_thread, NULL); + owner_started = 0; + return -1; + } + pthread_mutex_unlock(&state_lock); + return 77; +} + +int XFlush(Display *display) +{ + (void)display; + pthread_mutex_lock(&state_lock); + if (!owner_started) { + pthread_mutex_unlock(&state_lock); + return -1; + } + release_owner = 1; + pthread_cond_broadcast(&state_cond); + pthread_mutex_unlock(&state_lock); + if (pthread_join(owner_thread, NULL) != 0 || call.result != 0) { + return -1; + } + owner_started = 0; + return 78; +} diff --git a/tests/integration/kzt-bootstrap-tid.gdb b/tests/integration/kzt-bootstrap-tid.gdb new file mode 100644 index 0000000000..9364b99b16 --- /dev/null +++ b/tests/integration/kzt-bootstrap-tid.gdb @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Explicit diagnostic regression. Requires a LATX binary with debug symbols. +# Invoke with the existing native-thread opt-in fixture and its environment. +set pagination off +set confirm off +set print thread-events off +set $bootstrap_hits = 0 +break RunFunctionWithStateInternal if fnc == guest_allocate_tls_init +commands + silent + set $env = (CPUX86State *)lsenv->cpu_state + set $snapshot = (kzt_guest_parent_tls_snapshot_t *)$env->kzt_guest_tls_parent_snapshot + set $task = (TaskState *)((CPUState *)thread_cpu)->opaque + set $tid = *(unsigned int *)($env->segs[4].base + $snapshot->tid_offset) + if $tid != $task->ts_tid + printf "FAIL: Guest loader entered with tid=%u, expected=%u\n", $tid, $task->ts_tid + quit 1 + end + set $bootstrap_hits = $bootstrap_hits + 1 + continue +end +run +if $bootstrap_hits == 0 + echo FAIL: Guest loader bootstrap breakpoint was not reached\n + quit 1 +end +if $_exitcode != 0 + echo FAIL: native-thread fixture failed\n + quit 1 +end +printf "PASS: %d Guest loader entries had initialized TIDs\n", $bootstrap_hits diff --git a/tests/integration/kzt-cxx-tls-lifetime-destructor.cpp b/tests/integration/kzt-cxx-tls-lifetime-destructor.cpp new file mode 100644 index 0000000000..890aa023e3 --- /dev/null +++ b/tests/integration/kzt-cxx-tls-lifetime-destructor.cpp @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "kzt-cxx-tls-lifetime-shared.h" + +static int *external_destructors; + +ThreadGuard::~ThreadGuard() +{ + if (external_destructors) { + __atomic_add_fetch(external_destructors, 1, __ATOMIC_RELEASE); + } +} + +extern "C" void kzt_cxx_tls_helper_set_destructor_counter(int *counter) +{ + external_destructors = counter; +} diff --git a/tests/integration/kzt-cxx-tls-lifetime-guest.c b/tests/integration/kzt-cxx-tls-lifetime-guest.c new file mode 100644 index 0000000000..5763a36421 --- /dev/null +++ b/tests/integration/kzt-cxx-tls-lifetime-guest.c @@ -0,0 +1,126 @@ +#define _GNU_SOURCE +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include +#include + +#include + +#include "x11-async-bridge-values.h" + +typedef int (*plugin_check_fn)(void); +typedef void (*plugin_set_counters_fn)(int *, int *); + +static plugin_check_fn plugin_check; +typedef struct callback_control { + int ready; + int release; + int error; +} callback_control; + +static callback_control control; +static int constructor_count; +static int destructor_count; + +static Bool lifetime_callback(Display *display, xReply *reply, + char *buffer, int length, + XPointer opaque) +{ + callback_control *state = (callback_control *)opaque; + + (void)display; + if (!reply || !buffer || length != ASYNC_PROBE_REPLY_LENGTH || + !plugin_check || plugin_check() != 0) { + state->error = 1; + } + __atomic_store_n(&state->ready, 1, __ATOMIC_RELEASE); + while (!__atomic_load_n(&state->release, __ATOMIC_ACQUIRE)) { + sched_yield(); + } + return False; +} + +int main(int argc, char **argv) +{ + Display *display; + _XAsyncHandler handler = { 0 }; + plugin_set_counters_fn set_counters; + void *handle; + void *still_loaded; + + if (argc != 2) { + return 2; + } + display = calloc(1, sizeof(*display)); + if (!display) { + return 3; + } + handle = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!handle) { + fprintf(stderr, "FAIL: cannot load C++ TLS plugin: %s\n", + dlerror()); + return 4; + } + plugin_check = (plugin_check_fn)dlsym( + handle, "kzt_cxx_tls_lifetime_check"); + set_counters = (plugin_set_counters_fn)dlsym( + handle, "kzt_cxx_tls_lifetime_set_counters"); + if (!plugin_check || !set_counters) { + return 5; + } + set_counters(&constructor_count, &destructor_count); + + handler.handler = lifetime_callback; + handler.data = (XPointer)&control; + display->async_handlers = &handler; + if (XEventsQueued(display, QueuedAfterReading) != + ASYNC_PROBE_EVENTS_RETURN) { + return 6; + } + while (!__atomic_load_n(&control.ready, __ATOMIC_ACQUIRE)) { + sched_yield(); + } + if (control.error || constructor_count != 1 || + destructor_count != 0) { + fprintf(stderr, + "FAIL: C++ TLS callback setup error=%d ctor=%d dtor=%d\n", + control.error, constructor_count, destructor_count); + return 7; + } + if (dlclose(handle) != 0) { + return 8; + } + handle = NULL; + still_loaded = dlopen(argv[1], RTLD_LAZY | RTLD_NOLOAD); + if (!still_loaded) { + fprintf(stderr, + "FAIL: C++ TLS owner unloaded before destructor\n"); + return 9; + } + dlclose(still_loaded); + if (XFlush(display) != ASYNC_PROBE_FLUSH_RETURN) { + return 10; + } + if (control.error || constructor_count != 1 || + destructor_count != 1) { + fprintf(stderr, + "FAIL: C++ TLS teardown error=%d ctor=%d dtor=%d\n", + control.error, constructor_count, destructor_count); + return 11; + } + still_loaded = dlopen( + argv[1], RTLD_LAZY | RTLD_NOLOAD); + if (still_loaded) { + dlclose(still_loaded); + fprintf(stderr, + "FAIL: C++ TLS plugin remained loaded after destructor\n"); + return 12; + } + free(display); + fputs("PASS: C++ thread_local DSO survived until Host-thread exit\n", + stderr); + return 0; +} diff --git a/tests/integration/kzt-cxx-tls-lifetime-plugin.cpp b/tests/integration/kzt-cxx-tls-lifetime-plugin.cpp new file mode 100644 index 0000000000..2bb11208af --- /dev/null +++ b/tests/integration/kzt-cxx-tls-lifetime-plugin.cpp @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "kzt-cxx-tls-lifetime-shared.h" + +static int *external_constructors; + +ThreadGuard::ThreadGuard() : value(0x1234) +{ + if (external_constructors) { + __atomic_add_fetch( + external_constructors, 1, __ATOMIC_RELEASE); + } +} + +static thread_local ThreadGuard guard; + +extern "C" void kzt_cxx_tls_lifetime_set_counters( + int *constructors, int *destructors) +{ + external_constructors = constructors; + kzt_cxx_tls_helper_set_destructor_counter(destructors); +} + +extern "C" int kzt_cxx_tls_lifetime_check(void) +{ + return guard.value == 0x1234 ? 0 : 1; +} diff --git a/tests/integration/kzt-cxx-tls-lifetime-probe.c b/tests/integration/kzt-cxx-tls-lifetime-probe.c new file mode 100644 index 0000000000..5baec92360 --- /dev/null +++ b/tests/integration/kzt-cxx-tls-lifetime-probe.c @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include + +#include + +#include "x11-async-bridge-values.h" + +typedef struct lifetime_call { + _XAsyncHandler *handler; + Display *display; + xReply reply; +} lifetime_call; + +typedef struct callback_control { + int ready; + int release; + int error; +} callback_control; + +static lifetime_call call; +static pthread_t worker; +static int worker_started; + +static void *run_lifetime_callback(void *opaque) +{ + lifetime_call *current = opaque; + + current->handler->handler( + current->display, ¤t->reply, + (char *)¤t->reply, ASYNC_PROBE_REPLY_LENGTH, + current->handler->data); + /* Re-enabling deferred cancellation is not itself a cancellation point. */ + pthread_testcancel(); + return NULL; +} + +int XEventsQueued(Display *display, int mode) +{ + (void)mode; + if (!display || !display->async_handlers || worker_started) { + return -1; + } + memset(&call, 0, sizeof(call)); + call.handler = display->async_handlers; + call.display = display; + if (pthread_create(&worker, NULL, run_lifetime_callback, + &call) != 0) { + return -1; + } + worker_started = 1; + return ASYNC_PROBE_EVENTS_RETURN; +} + +int XFlush(Display *display) +{ + callback_control *control; + void *thread_result = NULL; + + (void)display; + if (!worker_started || !call.handler || !call.handler->data) { + return -1; + } + control = (callback_control *)call.handler->data; + if (pthread_cancel(worker) != 0) { + return -1; + } + __atomic_store_n(&control->release, 1, __ATOMIC_RELEASE); + if (pthread_join(worker, &thread_result) != 0 || + thread_result != PTHREAD_CANCELED) { + return -1; + } + worker_started = 0; + return ASYNC_PROBE_FLUSH_RETURN; +} diff --git a/tests/integration/kzt-cxx-tls-lifetime-shared.h b/tests/integration/kzt-cxx-tls-lifetime-shared.h new file mode 100644 index 0000000000..645c7d821e --- /dev/null +++ b/tests/integration/kzt-cxx-tls-lifetime-shared.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef KZT_CXX_TLS_LIFETIME_SHARED_H +#define KZT_CXX_TLS_LIFETIME_SHARED_H + +class ThreadGuard { +public: + ThreadGuard(); + ~ThreadGuard(); + + int value; +}; + +extern "C" void kzt_cxx_tls_helper_set_destructor_counter(int *counter); + +#endif diff --git a/tests/integration/kzt-guest-tls-opt-in-plugin.c b/tests/integration/kzt-guest-tls-opt-in-plugin.c new file mode 100644 index 0000000000..195d0c67fc --- /dev/null +++ b/tests/integration/kzt-guest-tls-opt-in-plugin.c @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +static __thread int value = 19; + +int kzt_opt_in_tls_value(void) +{ + return value++; +} diff --git a/tests/integration/kzt-guest-tls-opt-in.c b/tests/integration/kzt-guest-tls-opt-in.c new file mode 100644 index 0000000000..34bbb13ff1 --- /dev/null +++ b/tests/integration/kzt-guest-tls-opt-in.c @@ -0,0 +1,273 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HOST_PROBE +typedef struct Worker { + Display *display; + int index; + int result; + int synchronize; +} Worker; + +static __thread int host_tls = 51; +static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +static int start; +static int finished; +static Worker parked; +static pthread_t parked_thread; +static int park_ready; +static int park_release; + +static void *invoke(void *opaque) +{ + Worker *worker = opaque; + _XAsyncHandler *handler = worker->display->async_handlers; + xReply reply = { 0 }; + + if (worker->synchronize) { + pthread_mutex_lock(&lock); + while (!start) { + pthread_cond_wait(&cond, &lock); + } + pthread_mutex_unlock(&lock); + if (start < 0) { + return NULL; + } + } + for (int iteration = 0; iteration < 3; ++iteration) { + host_tls = 51 + iteration; + if (!handler->handler(worker->display, &reply, NULL, + worker->index * 10 + iteration, + handler->data) || + host_tls != 51 + iteration) { + worker->result = -1; + break; + } + } + if (worker->synchronize) { + pthread_mutex_lock(&lock); + ++finished; + pthread_cond_broadcast(&cond); + while (finished != 2) { + pthread_cond_wait(&cond, &lock); + } + pthread_mutex_unlock(&lock); + } + return NULL; +} + +static void *park_worker(void *opaque) +{ + invoke(opaque); + pthread_mutex_lock(&lock); + park_ready = 1; + pthread_cond_broadcast(&cond); + while (!park_release) { + pthread_cond_wait(&cond, &lock); + } + pthread_mutex_unlock(&lock); + return NULL; +} + +int XFlush(Display *display) +{ + (void)display; + pthread_mutex_lock(&lock); + park_release = 1; + pthread_cond_broadcast(&cond); + pthread_mutex_unlock(&lock); + return pthread_join(parked_thread, NULL); +} + +int XEventsQueued(Display *display, int mode) +{ + Worker workers[2] = { { .display = display }, + { .display = display, .index = 1 } }; + pthread_t threads[2]; + + if (mode == 2) { + parked.display = display; + if (pthread_create(&parked_thread, NULL, park_worker, &parked)) { + return -2; + } + pthread_mutex_lock(&lock); + while (!park_ready) { + pthread_cond_wait(&cond, &lock); + } + pthread_mutex_unlock(&lock); + return parked.result; + } + if (!mode) { + invoke(&workers[0]); + return workers[0].result; + } + start = 0; + finished = 0; + workers[0].synchronize = 1; + workers[1].synchronize = 1; + if (pthread_create(&threads[0], NULL, invoke, &workers[0])) { + return -2; + } + if (pthread_create(&threads[1], NULL, invoke, &workers[1])) { + pthread_mutex_lock(&lock); + start = -1; + pthread_cond_broadcast(&cond); + pthread_mutex_unlock(&lock); + pthread_join(threads[0], NULL); + return -2; + } + pthread_mutex_lock(&lock); + start = 1; + pthread_cond_broadcast(&cond); + pthread_mutex_unlock(&lock); + pthread_join(threads[0], NULL); + pthread_join(threads[1], NULL); + return workers[0].result || workers[1].result ? -3 : 0; +} +#else +static __thread int guest_tls = 7; +static __thread int guest_child_mode; +static uintptr_t addresses[2]; +static int hits; +static int destructor_count; +static pthread_key_t key; +static int attached; +static int (*late_value)(void); + +static void *guest_child(void *opaque) +{ + guest_child_mode = 1; + if (XEventsQueued(opaque, 0) || guest_tls != 10) { + return (void *)(uintptr_t)1; + } + return NULL; +} + +static void destroy_value(void *value) +{ + if (value != &guest_tls || guest_tls != 10) { + abort(); + } + __atomic_add_fetch(&destructor_count, 1, __ATOMIC_RELAXED); +} + +static Bool callback(Display *display, xReply *reply, char *buffer, + int length, XPointer opaque) +{ + int index = length / 10; + int iteration = length % 10; + (void)display; + (void)reply; + (void)buffer; + (void)opaque; + + if (index < 0 || index > 1 || iteration > 2 || + guest_tls != 7 + iteration || tolower('A') != 'a') { + fprintf(stderr, "FAIL: callback index=%d iteration=%d tls=%d lower=%d\n", + index, iteration, guest_tls, tolower('A')); + return False; + } + if (guest_child_mode) { + ++guest_tls; + return True; + } + if (late_value && late_value() != 19 + iteration) { + return False; + } + if (attached && index == 0 && iteration == 0) { + pthread_t child; + void *child_result = NULL; + + if (pthread_create(&child, NULL, guest_child, display) || + pthread_join(child, &child_result) || child_result) { + return False; + } + } + if (attached && pthread_setspecific(key, &guest_tls)) { + fprintf(stderr, "FAIL: setspecific key=%u\n", (unsigned)key); + return False; + } + addresses[index] = (uintptr_t)&guest_tls; + ++guest_tls; + __atomic_add_fetch(&hits, 1, __ATOMIC_RELAXED); + return True; +} + +int main(int argc, char **argv) +{ + Display *display = calloc(1, sizeof(*display)); + _XAsyncHandler handler = { .handler = callback }; + uintptr_t main_address = (uintptr_t)&guest_tls; + void *late_handle = NULL; + int phases; + + if (!display || argc < 2 || argc > 3) { + return 2; + } + attached = !strcmp(argv[1], "attached"); + display->async_handlers = &handler; + if (attached) { + guest_tls = 77; + if (pthread_key_create(&key, destroy_value)) { + return 3; + } + } + phases = attached && argc == 3 ? 3 : 1; + if (phases > 1 && XEventsQueued(display, 2)) { + return 9; + } + for (int phase = 0; phase < phases; ++phase) { + int result; + + if (phase == 1) { + late_handle = dlopen(argv[2], RTLD_NOW | RTLD_LOCAL); + late_value = late_handle + ? (int (*)(void))dlsym(late_handle, "kzt_opt_in_tls_value") + : NULL; + if (!late_value) { + return 7; + } + } else if (phase == 2) { + late_value = NULL; + if (dlclose(late_handle)) { + return 8; + } + } + hits = 0; + destructor_count = 0; + result = XEventsQueued(display, attached); + if (result || hits != (attached ? 6 : 3)) { + fprintf(stderr, "FAIL: phase=%d native result=%d hits=%d\n", + phase, result, hits); + return 4; + } + if (attached) { + if (guest_tls != 77 || addresses[0] == addresses[1] || + addresses[0] == main_address || addresses[1] == main_address || + destructor_count != 2) { + return 5; + } + } else if (guest_tls != 10 || addresses[0] != main_address) { + return 6; + } + } + if (attached) { + if (phases > 1 && XFlush(display)) { + return 10; + } + pthread_key_delete(key); + } + free(display); + printf("PASS: %s Guest TLS callback\n", attached ? "attached" : "existing"); + return 0; +} +#endif diff --git a/tests/integration/kzt-host-thread-cxx-tls-plugin.cpp b/tests/integration/kzt-host-thread-cxx-tls-plugin.cpp new file mode 100644 index 0000000000..a1944932e8 --- /dev/null +++ b/tests/integration/kzt-host-thread-cxx-tls-plugin.cpp @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +static int constructor_count; +static int destructor_count; + +class ThreadGuard { +public: + ThreadGuard() : value(0x4567) + { + __atomic_add_fetch(&constructor_count, 1, __ATOMIC_RELAXED); + } + + ~ThreadGuard() + { + __atomic_add_fetch(&destructor_count, 1, __ATOMIC_RELAXED); + } + + int value; +}; + +static thread_local ThreadGuard guard; + +extern "C" int kzt_host_thread_cxx_tls_check(void) +{ + if (guard.value < 0x4567 || guard.value > 0x4667) { + return 96; + } + ++guard.value; + return 0; +} + +extern "C" int kzt_host_thread_cxx_tls_counts( + int *constructors, int *destructors) +{ + if (!constructors || !destructors) { + return -1; + } + *constructors = __atomic_load_n( + &constructor_count, __ATOMIC_ACQUIRE); + *destructors = __atomic_load_n( + &destructor_count, __ATOMIC_ACQUIRE); + return 0; +} diff --git a/tests/integration/kzt-host-thread-tls-guest-plugin-b.c b/tests/integration/kzt-host-thread-tls-guest-plugin-b.c new file mode 100644 index 0000000000..e274347aad --- /dev/null +++ b/tests/integration/kzt-host-thread-tls-guest-plugin-b.c @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +static int plugin_b_anchor; +static __thread long plugin_b_values[3] = { 301, 302, 303 }; +static __thread void *plugin_b_pointer = &plugin_b_anchor; +/* Force each callback to touch both ends of the replacement TLS block. */ +static __thread unsigned char volatile + plugin_b_lifetime_probe[2 * 1024 * 1024]; +static __thread int plugin_b_calls; + +int kzt_host_thread_tls_plugin_check(void) +{ + if (plugin_b_values[0] != 301 || plugin_b_values[2] != 303 || + plugin_b_pointer != &plugin_b_anchor || + plugin_b_lifetime_probe[0] != 0 || + plugin_b_lifetime_probe[ + sizeof(plugin_b_lifetime_probe) - 1] != 0 || + plugin_b_calls < 0 || plugin_b_calls > 3) { + return 111; + } + ++plugin_b_calls; + return 0; +} diff --git a/tests/integration/kzt-host-thread-tls-guest-plugin.c b/tests/integration/kzt-host-thread-tls-guest-plugin.c new file mode 100644 index 0000000000..e82fd03dbc --- /dev/null +++ b/tests/integration/kzt-host-thread-tls-guest-plugin.c @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +static int plugin_anchor; +/* Force each callback to perform the TLS load instead of folding the value. */ +__thread void *volatile plugin_tls_pointer = &plugin_anchor; +static __thread int plugin_tls_counter; +static __thread int plugin_constructor_marker; +/* Touch both ends so the lifetime check covers the full dynamic TLS block. */ +static __thread unsigned char volatile + plugin_tls_lifetime_probe[8 * 1024 * 1024]; +static int constructor_error; +static int constructor_marker_observed; +static int total_checks; + +static void __attribute__((constructor)) check_tls_constructor(void) +{ + constructor_error = + (plugin_tls_pointer != &plugin_anchor) | + ((plugin_tls_counter != 0) << 1) | + ((plugin_tls_lifetime_probe[0] != 0) << 2) | + ((plugin_tls_lifetime_probe[ + sizeof(plugin_tls_lifetime_probe) - 1] != 0) << 3); + plugin_constructor_marker = 0x5a5a; +} + +int kzt_host_thread_tls_plugin_check(void) +{ + int checks; + + if (constructor_error) { + return 130 + constructor_error; + } + if (plugin_tls_pointer != &plugin_anchor) { + return 139; + } + if (plugin_tls_counter < 0 || plugin_tls_counter > 3) { + return 49; + } + if (plugin_constructor_marker == 0x5a5a) { + __atomic_store_n( + &constructor_marker_observed, 1, __ATOMIC_RELEASE); + } + checks = __atomic_add_fetch(&total_checks, 1, __ATOMIC_ACQ_REL); + if (checks >= 4 && + !__atomic_load_n( + &constructor_marker_observed, __ATOMIC_ACQUIRE)) { + return 50; + } + plugin_tls_lifetime_probe[0] = (unsigned char)plugin_tls_counter; + plugin_tls_lifetime_probe[sizeof(plugin_tls_lifetime_probe) - 1] = + (unsigned char)plugin_tls_counter; + ++plugin_tls_counter; + return 0; +} diff --git a/tests/integration/kzt-host-thread-tls-ie-plugin.c b/tests/integration/kzt-host-thread-tls-ie-plugin.c new file mode 100644 index 0000000000..1878006a59 --- /dev/null +++ b/tests/integration/kzt-host-thread-tls-ie-plugin.c @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +static int plugin_anchor; +static int constructor_error; + +__thread int ie_value + __attribute__((tls_model("initial-exec"))) = 0x1357; +/* Force each callback to perform the TLS load instead of folding the value. */ +__thread void *volatile ie_pointer + __attribute__((tls_model("initial-exec"))) = &plugin_anchor; +static __thread int ie_bss + __attribute__((tls_model("initial-exec"))); +/* Keep the aligned TLS object and its end-byte accesses observable. */ +__thread unsigned char volatile ie_aligned[64] + __attribute__((aligned(64), tls_model("initial-exec"))); +static __thread int ie_calls + __attribute__((tls_model("initial-exec"))); + +static void __attribute__((constructor)) check_initial_exec_constructor(void) +{ + constructor_error = (ie_value != 0x1357) | + ((ie_pointer != &plugin_anchor) << 1) | + ((ie_bss != 0) << 2) | + ((((uintptr_t)ie_aligned & 63) != 0) << 3); + ie_value = 0x2468; +} + +int kzt_host_thread_tls_ie_plugin_check(void) +{ + if (constructor_error) { + return 100 + constructor_error; + } + if (ie_pointer != &plugin_anchor) { + return 91; + } + if (ie_value != 0x1357 && ie_value != 0x2468) { + return 124; + } + if (((uintptr_t)ie_aligned & 63) != 0) { + return 125; + } + if (!ie_calls) { + if (ie_bss != 0) { + return 92; + } + ie_bss = 0x369c; + } else if (ie_bss != 0x369c) { + return 93; + } + ie_aligned[0] = (unsigned char)ie_calls; + ie_aligned[sizeof(ie_aligned) - 1] = + (unsigned char)(ie_calls + 1); + ++ie_calls; + return 0; +} diff --git a/tests/integration/kzt-host-thread-tls-tlsdesc-plugin.c b/tests/integration/kzt-host-thread-tls-tlsdesc-plugin.c new file mode 100644 index 0000000000..2d239a6e3b --- /dev/null +++ b/tests/integration/kzt-host-thread-tls-tlsdesc-plugin.c @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +/* Force the pure-.tbss TLSDesc object to survive optimization. */ +static __thread unsigned char volatile tlsdesc_page[4096] + __attribute__((aligned(4096))); +static __thread int tlsdesc_calls; + +int kzt_host_thread_tls_tlsdesc_check(void) +{ + if (((uintptr_t)tlsdesc_page & 4095) != 0 || + tlsdesc_page[0] != 0 || + tlsdesc_page[sizeof(tlsdesc_page) - 1] != 0 || + tlsdesc_calls < 0 || tlsdesc_calls > 5) { + return 117; + } + ++tlsdesc_calls; + return 0; +} diff --git a/tests/integration/kzt-pthread-tsd-alias-guest.c b/tests/integration/kzt-pthread-tsd-alias-guest.c new file mode 100644 index 0000000000..9076bee26c --- /dev/null +++ b/tests/integration/kzt-pthread-tsd-alias-guest.c @@ -0,0 +1,216 @@ +#define _GNU_SOURCE +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include +#include + +#include "kzt-pthread-tsd-alias.h" +#include + +#define WORKER_COUNT 16 +#define DESTRUCTOR_PASSES 3 + +typedef struct worker_state { + pthread_key_t key; + uintptr_t expected; + int remaining; +} worker_state; + +static pthread_barrier_t start_barrier; +static pthread_key_t worker_keys[WORKER_COUNT]; +static int worker_errors[WORKER_COUNT]; +static int destructor_calls; +static int destructor_completions; + +static int check_guest_libc_symbol(const char *name, void *address) +{ + Dl_info info = { 0 }; + + if (!dladdr(address, &info) || !info.dli_fname || + (!strstr(info.dli_fname, "libc.so") && + !strstr(info.dli_fname, "libpthread.so"))) { + fprintf(stderr, + "FAIL: %s address=%p owner=%s\n", + name, address, + info.dli_fname ? info.dli_fname : ""); + return -1; + } + return 0; +} + +static void iterating_destructor(void *opaque) +{ + worker_state *state = opaque; + + __sync_fetch_and_add(&destructor_calls, 1); + if (--state->remaining > 0) { + if (pthread_setspecific(state->key, state) != 0) { + __sync_fetch_and_add(&worker_errors[0], 1); + } + return; + } + __sync_fetch_and_add(&destructor_completions, 1); + free(state); +} + +static void *run_worker(void *opaque) +{ + size_t index = (size_t)(uintptr_t)opaque; + worker_state *state = calloc(1, sizeof(*state)); + int result; + + if (!state) { + worker_errors[index] = 1; + return NULL; + } + state->expected = UINT64_C(0x10000) + index; + state->remaining = DESTRUCTOR_PASSES; + pthread_barrier_wait(&start_barrier); + + if (index & 1) { + result = pthread_key_create(&state->key, iterating_destructor); + } else { + result = direct_pthread_key_create( + &state->key, iterating_destructor); + } + if (result != 0) { + worker_errors[index] = 2; + free(state); + return NULL; + } + worker_keys[index] = state->key; + + if (index & 1) { + result = direct_pthread_setspecific(state->key, state); + } else { + result = pthread_setspecific(state->key, state); + } + if (result != 0 || + pthread_getspecific(state->key) != state || + direct_pthread_getspecific(state->key) != state) { + worker_errors[index] = 3; + return NULL; + } + + if ((index & 1) + ? pthread_setspecific(state->key, NULL) != 0 + : direct_pthread_setspecific(state->key, NULL) != 0) { + worker_errors[index] = 4; + return NULL; + } + if (pthread_getspecific(state->key) != NULL || + direct_pthread_getspecific(state->key) != NULL) { + worker_errors[index] = 5; + return NULL; + } + if ((index & 1) + ? direct_pthread_setspecific(state->key, state) != 0 + : pthread_setspecific(state->key, state) != 0) { + worker_errors[index] = 6; + } + return NULL; +} + +int main(int argc, char **argv) +{ + pthread_t workers[WORKER_COUNT]; + pthread_key_t first_key; + pthread_key_t reused_key; + uintptr_t main_value = UINT64_C(0x12345678); + int run_workers; + + if (check_guest_libc_symbol( + "pthread_getspecific", (void *)pthread_getspecific) != 0 || + check_guest_libc_symbol( + "pthread_setspecific", (void *)pthread_setspecific) != 0) { + return 11; + } + + if (argc == 1) { + run_workers = 1; + } else if (argc == 2 && + strcmp(argv[1], "--single-thread") == 0) { + run_workers = 0; + } else { + return 2; + } + if (run_workers) { + if (pthread_barrier_init(&start_barrier, NULL, + WORKER_COUNT + 1) != 0) { + return 2; + } + for (size_t index = 0; index < WORKER_COUNT; ++index) { + int create_result = pthread_create( + &workers[index], NULL, run_worker, + (void *)(uintptr_t)index); + + if (create_result != 0) { + fprintf(stderr, + "FAIL: Guest pthread_create index=%zu error=%d\n", + index, create_result); + return 3; + } + } + pthread_barrier_wait(&start_barrier); + for (size_t index = 0; index < WORKER_COUNT; ++index) { + if (pthread_join(workers[index], NULL) != 0) { + return 4; + } + } + pthread_barrier_destroy(&start_barrier); + + for (size_t index = 0; index < WORKER_COUNT; ++index) { + if (worker_errors[index]) { + fprintf(stderr, "FAIL: Guest TSD worker %zu error=%d\n", + index, worker_errors[index]); + return 5; + } + for (size_t other = 0; other < index; ++other) { + if (worker_keys[index] == worker_keys[other]) { + fprintf(stderr, + "FAIL: concurrent Guest TSD key collision\n"); + return 6; + } + } + } + if (destructor_calls != WORKER_COUNT * DESTRUCTOR_PASSES || + destructor_completions != WORKER_COUNT) { + fprintf(stderr, + "FAIL: Guest TSD destructors calls=%d " + "completions=%d\n", + destructor_calls, destructor_completions); + return 7; + } + for (size_t index = 0; index < WORKER_COUNT; ++index) { + if (pthread_key_delete(worker_keys[index]) != 0) { + return 8; + } + } + } + + if (pthread_key_create(&first_key, NULL) != 0 || + direct_pthread_setspecific(first_key, &main_value) != 0 || + pthread_getspecific(first_key) != &main_value || + direct_pthread_getspecific(first_key) != &main_value || + pthread_key_delete(first_key) != 0 || + direct_pthread_key_create(&reused_key, NULL) != 0) { + fprintf(stderr, "FAIL: Guest TSD public/private alias matrix\n"); + return 9; + } + if (reused_key != first_key || + pthread_getspecific(reused_key) != NULL || + direct_pthread_getspecific(reused_key) != NULL || + pthread_setspecific(reused_key, &main_value) != 0 || + direct_pthread_getspecific(reused_key) != &main_value || + pthread_key_delete(reused_key) != 0) { + fprintf(stderr, "FAIL: Guest TSD key reuse retained stale state\n"); + return 10; + } + + fputs("PASS: Guest pthread TSD remains Guest-libc authoritative\n", + stderr); + return 0; +} diff --git a/tests/integration/kzt-pthread-tsd-alias.h b/tests/integration/kzt-pthread-tsd-alias.h new file mode 100644 index 0000000000..77c15c1f3d --- /dev/null +++ b/tests/integration/kzt-pthread-tsd-alias.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef KZT_PTHREAD_TSD_ALIAS_H +#define KZT_PTHREAD_TSD_ALIAS_H + +#include + +int direct_pthread_key_create(pthread_key_t *key, + void (*destructor)(void *)); +int direct_pthread_setspecific(pthread_key_t key, + const void *value); +void *direct_pthread_getspecific(pthread_key_t key); + +__asm__(".symver direct_pthread_key_create," + "__pthread_key_create@GLIBC_2.2.5"); +__asm__(".symver direct_pthread_setspecific," + "__pthread_setspecific@GLIBC_2.2.5"); +__asm__(".symver direct_pthread_getspecific," + "__pthread_getspecific@GLIBC_2.2.5"); + +#endif diff --git a/tests/integration/kzt-tls-dlopen-stress-guest.c b/tests/integration/kzt-tls-dlopen-stress-guest.c new file mode 100644 index 0000000000..ba7eef373d --- /dev/null +++ b/tests/integration/kzt-tls-dlopen-stress-guest.c @@ -0,0 +1,553 @@ +#define _GNU_SOURCE +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "x11-async-bridge-values.h" +#include "kzt-tls-dlopen-stress-shared.h" + +#define STRESS_ITERATIONS 64 +#define FORK_FUTEX_ITERATIONS 16 + +typedef int (*plugin_check_fn)(void); + +static plugin_check_fn plugin_check; +static const char *attached_plugin_path; +static const char *concurrent_plugin_paths[2]; +static size_t plugin_module_id; +static int callback_error; +static int callback_hits; +static int run_attached_dlopen; +static unsigned int concurrent_plugin_index; +static int fork_child_after_dlopen; +static int fmt_callback_hits; + +typedef struct fork_futex_state { + _Atomic uint32_t child_word; + _Atomic uint32_t main_word; + _Atomic uint32_t worker_ready; + _Atomic uint32_t child_stage; +} fork_futex_state; + +static fork_futex_state *fork_state; + +static int exercise_fork_child_semantics(void) +{ + pthread_key_t key; + void *key_value = (void *)(uintptr_t)0x1357; + locale_t locale; + locale_t previous; + + atomic_store_explicit( + &fork_state->child_stage, 1, memory_order_release); + if (pthread_key_create(&key, NULL) != 0 || + pthread_setspecific(key, key_value) != 0 || + pthread_getspecific(key) != key_value || + pthread_key_delete(key) != 0) { + return -1; + } + atomic_store_explicit( + &fork_state->child_stage, 2, memory_order_release); + if (!setlocale(LC_NUMERIC, NULL)) { + return -1; + } + atomic_store_explicit( + &fork_state->child_stage, 3, memory_order_release); + locale = newlocale(LC_ALL_MASK, "C", NULL); + if (!locale) { + return -1; + } + previous = uselocale(locale); + if (!previous || !uselocale(previous)) { + freelocale(locale); + return -1; + } + freelocale(locale); + atomic_store_explicit( + &fork_state->child_stage, 4, memory_order_release); + return 0; +} + +static int stress_futex_wait(_Atomic uint32_t *word, uint32_t expected, + int private) +{ + int operation = FUTEX_WAIT | (private ? FUTEX_PRIVATE_FLAG : 0); + + return syscall(SYS_futex, word, operation, expected, + NULL, NULL, 0); +} + +static int stress_futex_wait_timed( + _Atomic uint32_t *word, uint32_t expected, int private) +{ + const struct timespec timeout = { + .tv_sec = 0, + .tv_nsec = 10 * 1000 * 1000, + }; + int operation = FUTEX_WAIT | (private ? FUTEX_PRIVATE_FLAG : 0); + + return syscall(SYS_futex, word, operation, expected, + &timeout, NULL, 0); +} + +static void stress_futex_wake(_Atomic uint32_t *word, int private) +{ + int operation = FUTEX_WAKE | (private ? FUTEX_PRIVATE_FLAG : 0); + + (void)syscall(SYS_futex, word, operation, 1, NULL, NULL, 0); +} + +static void *run_fork_futex_worker(void *opaque) +{ + (void)opaque; + atomic_store_explicit( + &fork_state->worker_ready, 1, memory_order_release); + stress_futex_wake(&fork_state->worker_ready, 0); + while (atomic_load_explicit( + &fork_state->child_word, + memory_order_acquire) == UINT32_C(0x80000000)) { + if (stress_futex_wait( + &fork_state->child_word, UINT32_C(0x80000000), 0) != 0 && + errno != EAGAIN && errno != EINTR) { + return (void *)1; + } + } + atomic_store_explicit( + &fork_state->main_word, 1, memory_order_release); + stress_futex_wake(&fork_state->main_word, 1); + return NULL; +} + +static int run_fork_futex_child(const char *fd_text, + const char *plugin_path, + const char *expected_runtime_root) +{ + const char *runtime_root = getenv("LAT_LD_PREFIX"); + pthread_key_t key; + void *key_value = (void *)(uintptr_t)0x2468; + void *handle; + plugin_check_fn check; + int fd = 0; + + if (!fd_text[0] || !plugin_path || !expected_runtime_root || + !runtime_root || strcmp(runtime_root, expected_runtime_root) != 0) { + return -1; + } + for (const char *cursor = fd_text; *cursor; ++cursor) { + if (*cursor < '0' || *cursor > '9' || + fd > (INT32_MAX - (*cursor - '0')) / 10) { + return -1; + } + fd = fd * 10 + (*cursor - '0'); + } + fork_state = mmap(NULL, sizeof(*fork_state), + PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (fork_state == MAP_FAILED) { + return -1; + } + if (pthread_key_create(&key, NULL) != 0 || + pthread_setspecific(key, key_value) != 0 || + pthread_getspecific(key) != key_value) { + return -1; + } + handle = dlopen(plugin_path, RTLD_NOW | RTLD_LOCAL); + check = handle ? (plugin_check_fn)dlsym( + handle, "kzt_host_thread_tls_plugin_check") + : NULL; + if (!handle || !check || check() != 0 || dlclose(handle) != 0 || + pthread_key_delete(key) != 0) { + return -1; + } + atomic_store_explicit( + &fork_state->child_word, 0, memory_order_release); + stress_futex_wake(&fork_state->child_word, 0); + return munmap(fork_state, sizeof(*fork_state)); +} + +static int run_fork_futex_topology(const char *program_path, + const char *plugin_path) +{ + const char *runtime_root = getenv("LAT_LD_PREFIX"); + int shared_fd = syscall( + SYS_memfd_create, "kzt-fork-futex", 0); + + if (!program_path || !plugin_path || !runtime_root || shared_fd < 0 || + ftruncate(shared_fd, sizeof(*fork_state)) != 0) { + return -1; + } + fork_state = mmap(NULL, sizeof(*fork_state), + PROT_READ | PROT_WRITE, + MAP_SHARED, shared_fd, 0); + if (fork_state == MAP_FAILED) { + return -1; + } + for (int iteration = 0; + iteration < FORK_FUTEX_ITERATIONS; ++iteration) { + pthread_t thread; + pid_t child; + pid_t waited = 0; + int status; + int wait_timeouts = 0; + void *thread_result = NULL; + + atomic_store_explicit( + &fork_state->child_word, + UINT32_C(0x80000000), memory_order_relaxed); + atomic_store_explicit( + &fork_state->main_word, 0, memory_order_relaxed); + atomic_store_explicit( + &fork_state->worker_ready, 0, memory_order_relaxed); + atomic_store_explicit( + &fork_state->child_stage, 0, memory_order_relaxed); + if (pthread_create( + &thread, NULL, run_fork_futex_worker, NULL) != 0) { + return -1; + } + while (!atomic_load_explicit( + &fork_state->worker_ready, memory_order_acquire)) { + if (stress_futex_wait( + &fork_state->worker_ready, 0, 0) != 0 && + errno != EAGAIN && errno != EINTR) { + return -1; + } + } + child = fork(); + if (child == 0) { + if (iteration & 1) { + char fd_text[24]; + + snprintf(fd_text, sizeof(fd_text), "%d", shared_fd); + execl(program_path, program_path, + "--fork-futex-child", fd_text, + plugin_path, runtime_root, NULL); + _exit(127); + } else { + if (exercise_fork_child_semantics() != 0) { + _exit(20); + } + atomic_store_explicit( + &fork_state->child_word, 0, memory_order_release); + stress_futex_wake(&fork_state->child_word, 0); + _exit(0); + } + } + if (child < 0) { + return -1; + } + while (!atomic_load_explicit( + &fork_state->main_word, memory_order_acquire) && + wait_timeouts < 200) { + int futex_result = stress_futex_wait_timed( + &fork_state->main_word, 0, 1); + + if (futex_result != 0 && errno != EAGAIN && + errno != EINTR && errno != ETIMEDOUT) { + return -1; + } + if (futex_result != 0 && errno == ETIMEDOUT) { + ++wait_timeouts; + } + if (!waited) { + waited = waitpid(child, &status, WNOHANG); + if (waited < 0) { + return -1; + } + } + } + if (!atomic_load_explicit( + &fork_state->main_word, memory_order_acquire) || + pthread_join(thread, &thread_result) != 0 || thread_result || + (!waited && waitpid(child, &status, 0) != child) || + !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fprintf(stderr, + "FAIL: fork semantic child iteration=%d stage=%u\n", + iteration, atomic_load_explicit( + &fork_state->child_stage, + memory_order_acquire)); + return -1; + } + } + if (munmap(fork_state, sizeof(*fork_state)) != 0) { + return -1; + } + return close(shared_fd); +} + +void kzt_tls_stress_mark_fork_child(void) +{ + fork_child_after_dlopen = 1; +} + +typedef struct guest_dtv_entry { + uintptr_t value; + uintptr_t to_free; +} guest_dtv_entry; + +static guest_dtv_entry *read_guest_dtv(void) +{ + guest_dtv_entry *dtv; + + __asm__ volatile("movq %%fs:8, %0" : "=r"(dtv)); + return dtv; +} + +static Bool stress_callback(Display *display, xReply *reply, + char *buffer, int length, + XPointer opaque) +{ + (void)display; + (void)opaque; + if (!reply || !buffer || length != ASYNC_PROBE_REPLY_LENGTH) { + __sync_val_compare_and_swap(&callback_error, 0, 1); + } else if (run_attached_dlopen) { + const char *path = run_attached_dlopen == 2 + ? concurrent_plugin_paths[ + __sync_fetch_and_add(&concurrent_plugin_index, 1) & 1] + : attached_plugin_path; + void *handle = dlopen( + path, RTLD_NOW | RTLD_LOCAL); + plugin_check_fn attached_check = handle + ? (plugin_check_fn)dlsym( + handle, "kzt_host_thread_tls_plugin_check") + : NULL; + + if (!handle) { + const char *loader_error = dlerror(); + + fprintf(stderr, + "FAIL: attached dlopen path=%s: %s\n", path, + loader_error ? loader_error : "no loader error"); + __sync_val_compare_and_swap(&callback_error, 0, 4); + } else if (!attached_check) { + const char *loader_error = dlerror(); + + fprintf(stderr, + "FAIL: attached dlsym path=%s: %s\n", path, + loader_error ? loader_error : "no loader error"); + __sync_val_compare_and_swap(&callback_error, 0, 5); + } else if (attached_check() != 0) { + __sync_val_compare_and_swap(&callback_error, 0, 6); + } else if (dlclose(handle) != 0) { + __sync_val_compare_and_swap(&callback_error, 0, 7); + } + } else if (plugin_check) { + if (plugin_check() != 0) { + __sync_val_compare_and_swap(&callback_error, 0, 2); + } + } else if (plugin_module_id) { + guest_dtv_entry *dtv = read_guest_dtv(); + + if (dtv[plugin_module_id].value != 0 || + dtv[plugin_module_id].to_free != 0) { + __sync_val_compare_and_swap(&callback_error, 0, 3); + } + } + __sync_fetch_and_add(&callback_hits, 1); + return False; +} + +static int stress_fmt_callback(Display *display) +{ + __sync_fetch_and_add(&fmt_callback_hits, 1); + return XNoOp(display); +} + +static int run_cycle(Display *display, const char *path) +{ + void *handle = dlopen(path, RTLD_NOW | RTLD_LOCAL); + + if (!handle) { + return -1; + } + if (fork_child_after_dlopen) { + (void)dlerror(); + _exit(0); + } + plugin_check = (plugin_check_fn)dlsym( + handle, "kzt_host_thread_tls_plugin_check"); + if (!plugin_check || + dlinfo(handle, RTLD_DI_TLS_MODID, &plugin_module_id) != 0 || + !plugin_module_id || + XEventsQueued(display, QueuedAfterReading) != + ASYNC_PROBE_EVENTS_RETURN || callback_error) { + return -1; + } + plugin_check = NULL; + if (dlclose(handle) != 0 || + XEventsQueued(display, QueuedAfterReading) != + ASYNC_PROBE_EVENTS_RETURN || callback_error) { + return -1; + } + return 0; +} + +int main(int argc, char **argv) +{ + Display *display; + _XAsyncHandler handler = { 0 }; + size_t retired_ie_module_id; + void *wrapped_handles[4]; + void *normal_handle; + void *noload_handle; + plugin_check_fn noload_check; + void *wrapped_unloaded_handle; + void *wrapped_noload_handle; + + if (argc == 5 && strcmp(argv[1], "--fork-futex-child") == 0) { + return run_fork_futex_child( + argv[2], argv[3], argv[4]) == 0 ? 0 : 19; + } + if (argc != 4) { + return 2; + } + display = calloc(1, sizeof(*display)); + if (!display) { + return 3; + } + handler.handler = stress_callback; + display->async_handlers = &handler; + if (XEventsQueued(display, 101) != ASYNC_PROBE_EVENTS_RETURN) { + return 21; + } + if (run_fork_futex_topology(argv[0], argv[1]) != 0) { + (void)XEventsQueued(display, 102); + return 18; + } + if (XEventsQueued(display, 102) != ASYNC_PROBE_EVENTS_RETURN) { + return 21; + } + for (size_t index = 0; index < 4; ++index) { + wrapped_handles[index] = dlopen( + "libX11.so.6", RTLD_NOW | RTLD_LOCAL); + if (!wrapped_handles[index]) { + return 11; + } + } + for (size_t index = 0; index < 4; ++index) { + if (dlclose(wrapped_handles[index]) != 0) { + return 12; + } + } + attached_plugin_path = argv[1]; + run_attached_dlopen = 1; + (void)XSetAfterFunction(display, stress_fmt_callback); + if (XEventsQueued(display, 100) != ASYNC_PROBE_EVENTS_RETURN || + callback_error || fmt_callback_hits != 1) { + fprintf(stderr, + "FAIL: RunFunctionFmt overlapped Guest TLS propagation " + "error=%d hits=%d\n", + callback_error, fmt_callback_hits); + return 20; + } + run_attached_dlopen = 0; + wrapped_unloaded_handle = dlopen( + "libxcb.so.1", RTLD_NOW | RTLD_LOCAL); + if (!wrapped_unloaded_handle || + dlclose(wrapped_unloaded_handle) != 0) { + return 16; + } + wrapped_noload_handle = dlopen( + "libxcb.so.1", + RTLD_NOW | RTLD_LOCAL | RTLD_NOLOAD); + if (wrapped_noload_handle) { + return 17; + } + if (XEventsQueued(display, QueuedAfterReading) != + ASYNC_PROBE_EVENTS_RETURN || callback_error) { + return 13; + } + if (run_cycle(display, argv[3]) != 0) { + return 4; + } + normal_handle = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + noload_handle = dlopen( + argv[1], RTLD_NOW | RTLD_LOCAL | RTLD_NOLOAD); + noload_check = normal_handle + ? (plugin_check_fn)dlsym( + normal_handle, "kzt_host_thread_tls_plugin_check") + : NULL; + if (!normal_handle || !noload_handle || !noload_check || + dlclose(noload_handle) != 0 || noload_check() != 0 || + dlclose(normal_handle) != 0) { + return 14; + } + retired_ie_module_id = plugin_module_id; + if (run_cycle(display, argv[1]) != 0 || + plugin_module_id != retired_ie_module_id || + run_cycle(display, argv[2]) != 0) { + return 4; + } + callback_hits = 0; + + for (int iteration = 0; iteration < STRESS_ITERATIONS; ++iteration) { + const char *path = argv[1 + (iteration & 1)]; + + if (run_cycle(display, path) != 0) { + fprintf(stderr, "FAIL: stress dlopen iteration=%d: %s\n", + iteration, dlerror()); + return 5; + } + } + run_attached_dlopen = 1; + for (int iteration = 0; iteration < STRESS_ITERATIONS; ++iteration) { + attached_plugin_path = argv[1 + (iteration & 1)]; + if (XEventsQueued(display, QueuedAfterReading) != + ASYNC_PROBE_EVENTS_RETURN || callback_error) { + fprintf(stderr, + "FAIL: attached stress dlopen iteration=%d error=%d\n", + iteration, callback_error); + return 6; + } + } + run_attached_dlopen = 0; + if (XEventsQueued(display, 99) != ASYNC_PROBE_EVENTS_RETURN || + callback_error || callback_hits != 3 * STRESS_ITERATIONS + 2) { + fprintf(stderr, + "FAIL: concurrent attached warmup hits=%d error=%d\n", + callback_hits, callback_error); + return 9; + } + run_attached_dlopen = 2; + concurrent_plugin_paths[0] = argv[1]; + concurrent_plugin_paths[1] = argv[2]; + for (int iteration = 0; iteration < STRESS_ITERATIONS / 2; + ++iteration) { + if (XEventsQueued(display, 99) != + ASYNC_PROBE_EVENTS_RETURN || callback_error) { + fprintf(stderr, + "FAIL: concurrent attached dlopen iteration=%d " + "error=%d\n", iteration, callback_error); + return 10; + } + } + if (XFlush(display) != ASYNC_PROBE_FLUSH_RETURN) { + return 7; + } + if (callback_hits != 4 * STRESS_ITERATIONS + 2) { + fprintf(stderr, + "FAIL: stress hits=%d\n", callback_hits); + return 8; + } + free(display); + fputs("PASS: attached Guest TLS survived 64 A/B dlopen cycles " + "from Guest, Host-attached, and concurrent threads\n", stderr); + return 0; +} diff --git a/tests/integration/kzt-tls-dlopen-stress-ie-plugin.c b/tests/integration/kzt-tls-dlopen-stress-ie-plugin.c new file mode 100644 index 0000000000..20d87e746d --- /dev/null +++ b/tests/integration/kzt-tls-dlopen-stress-ie-plugin.c @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include + +#include "kzt-tls-dlopen-stress-shared.h" + +static int anchor; +static int constructor_error; +static pid_t constructor_child_pid = -1; +static int constructor_is_fork_child; + +__thread int stress_ie_value + __attribute__((tls_model("initial-exec"))) = 0x2468; +__thread void *stress_ie_pointer + __attribute__((tls_model("initial-exec"))) = &anchor; + +static void *constructor_loader_query(void *opaque) +{ + (void)opaque; + (void)dlerror(); + return NULL; +} + +static void __attribute__((constructor)) stress_ie_constructor(void) +{ + pthread_t thread; + pid_t child; + + if (pthread_create( + &thread, NULL, constructor_loader_query, NULL) != 0 || + pthread_join(thread, NULL) != 0) { + constructor_error = 1; + } + child = fork(); + if (child == 0) { + constructor_is_fork_child = 1; + (void)dlerror(); + kzt_tls_stress_mark_fork_child(); + return; + } + if (child < 0) { + constructor_error = 1; + } else { + constructor_child_pid = child; + } +} + +int kzt_host_thread_tls_plugin_check(void) +{ + int status; + + if (constructor_is_fork_child) { + _exit(0); + } + if (constructor_child_pid > 0) { + if (waitpid(constructor_child_pid, &status, 0) != + constructor_child_pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) { + constructor_error = 1; + } + constructor_child_pid = -1; + } + return !constructor_error && stress_ie_value == 0x2468 && + stress_ie_pointer == &anchor ? 0 : 1; +} diff --git a/tests/integration/kzt-tls-dlopen-stress-probe.c b/tests/integration/kzt-tls-dlopen-stress-probe.c new file mode 100644 index 0000000000..95e5ffc5a3 --- /dev/null +++ b/tests/integration/kzt-tls-dlopen-stress-probe.c @@ -0,0 +1,324 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include + +#include + +#include "x11-async-bridge-values.h" + +typedef struct persistent_worker { + pthread_mutex_t lock; + pthread_cond_t ready; + pthread_cond_t done; + pthread_t thread; + _XAsyncHandler *handler; + Display *display; + unsigned int generation; + unsigned int completed_generation; + int started; + int shutdown; +} persistent_worker; + +static persistent_worker worker = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .ready = PTHREAD_COND_INITIALIZER, + .done = PTHREAD_COND_INITIALIZER, +}; +static persistent_worker concurrent_workers[2]; +static int concurrent_started; +static pthread_mutex_t fmt_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t fmt_cond = PTHREAD_COND_INITIALIZER; +static int (*fmt_callback)(Display *display); +static Display *fmt_display; +static int fmt_entered; +static int fmt_release; +static int fmt_loader_completed; +static int fmt_callback_result; +static pthread_t semantic_stress_thread; +static int semantic_stress_running; +static int semantic_stress_started; + +static void *run_semantic_stress(void *opaque) +{ + Display *display = opaque; + + while (__atomic_load_n(&semantic_stress_running, + __ATOMIC_ACQUIRE)) { + xReply reply; + + memset(&reply, 0, sizeof(reply)); + display->async_handlers->handler( + display, &reply, (char *)&reply, + ASYNC_PROBE_REPLY_LENGTH, + display->async_handlers->data); + } + return NULL; +} + +static int start_semantic_stress(Display *display) +{ + if (!display->async_handlers || semantic_stress_started) { + return -1; + } + semantic_stress_running = 1; + if (pthread_create(&semantic_stress_thread, NULL, + run_semantic_stress, display) != 0) { + semantic_stress_running = 0; + return -1; + } + semantic_stress_started = 1; + return ASYNC_PROBE_EVENTS_RETURN; +} + +static int stop_semantic_stress(void) +{ + if (!semantic_stress_started) { + return -1; + } + __atomic_store_n(&semantic_stress_running, 0, + __ATOMIC_RELEASE); + if (pthread_join(semantic_stress_thread, NULL) != 0) { + return -1; + } + semantic_stress_started = 0; + return ASYNC_PROBE_EVENTS_RETURN; +} + +static void *run_fmt_callback(void *opaque) +{ + (void)opaque; + fmt_callback_result = fmt_callback(fmt_display); + return NULL; +} + +static void *run_fmt_loader_callback(void *opaque) +{ + _XAsyncHandler *handler = opaque; + xReply reply; + + memset(&reply, 0, sizeof(reply)); + handler->handler(fmt_display, &reply, (char *)&reply, + ASYNC_PROBE_REPLY_LENGTH, handler->data); + pthread_mutex_lock(&fmt_lock); + fmt_loader_completed = 1; + pthread_cond_broadcast(&fmt_cond); + pthread_mutex_unlock(&fmt_lock); + return NULL; +} + +static int run_fmt_propagation_race(Display *display) +{ + const struct timespec wait_duration = { + .tv_sec = 0, + .tv_nsec = 100 * 1000 * 1000, + }; + struct timespec deadline; + pthread_t callback_thread; + pthread_t loader_thread; + int completed_before_release; + int wait_result; + + if (!fmt_callback || !display->async_handlers) { + return -1; + } + pthread_mutex_lock(&fmt_lock); + fmt_display = display; + fmt_entered = 0; + fmt_release = 0; + fmt_loader_completed = 0; + fmt_callback_result = -1; + if (pthread_create(&callback_thread, NULL, + run_fmt_callback, NULL) != 0) { + pthread_mutex_unlock(&fmt_lock); + return -1; + } + while (!fmt_entered) { + pthread_cond_wait(&fmt_cond, &fmt_lock); + } + if (pthread_create(&loader_thread, NULL, + run_fmt_loader_callback, + display->async_handlers) != 0) { + fmt_release = 1; + pthread_cond_broadcast(&fmt_cond); + pthread_mutex_unlock(&fmt_lock); + pthread_join(callback_thread, NULL); + return -1; + } + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += wait_duration.tv_sec; + deadline.tv_nsec += wait_duration.tv_nsec; + if (deadline.tv_nsec >= 1000 * 1000 * 1000) { + ++deadline.tv_sec; + deadline.tv_nsec -= 1000 * 1000 * 1000; + } + wait_result = 0; + while (!fmt_loader_completed && wait_result == 0) { + wait_result = pthread_cond_timedwait( + &fmt_cond, &fmt_lock, &deadline); + } + completed_before_release = fmt_loader_completed; + fmt_release = 1; + pthread_cond_broadcast(&fmt_cond); + pthread_mutex_unlock(&fmt_lock); + + if (pthread_join(loader_thread, NULL) != 0 || + pthread_join(callback_thread, NULL) != 0 || + fmt_callback_result != 0) { + return -1; + } + return completed_before_release ? -2 + : ASYNC_PROBE_EVENTS_RETURN; +} + +int (*XSetAfterFunction(Display *display, + int (*callback)(Display *)))(Display *) +{ + int (*previous)(Display *) = fmt_callback; + + fmt_display = display; + fmt_callback = callback; + return previous; +} + +int XNoOp(Display *display) +{ + (void)display; + pthread_mutex_lock(&fmt_lock); + fmt_entered = 1; + pthread_cond_broadcast(&fmt_cond); + while (!fmt_release) { + pthread_cond_wait(&fmt_cond, &fmt_lock); + } + pthread_mutex_unlock(&fmt_lock); + return 0; +} + +static void *run_worker(void *opaque) +{ + persistent_worker *state = opaque; + + pthread_mutex_lock(&state->lock); + while (!state->shutdown) { + xReply reply; + unsigned int generation; + + while (!state->shutdown && + state->completed_generation == state->generation) { + pthread_cond_wait(&state->ready, &state->lock); + } + if (state->shutdown) { + break; + } + generation = state->generation; + memset(&reply, 0, sizeof(reply)); + pthread_mutex_unlock(&state->lock); + state->handler->handler( + state->display, &reply, (char *)&reply, + ASYNC_PROBE_REPLY_LENGTH, state->handler->data); + pthread_mutex_lock(&state->lock); + state->completed_generation = generation; + pthread_cond_signal(&state->done); + } + pthread_mutex_unlock(&state->lock); + return NULL; +} + +int XEventsQueued(Display *display, int mode) +{ + if (mode == 101) { + return start_semantic_stress(display); + } + if (mode == 102) { + return stop_semantic_stress(); + } + if (mode == 100) { + return run_fmt_propagation_race(display); + } + if (mode == 99) { + if (!concurrent_started) { + concurrent_started = 1; + for (size_t index = 0; index < 2; ++index) { + persistent_worker *state = &concurrent_workers[index]; + + pthread_mutex_init(&state->lock, NULL); + pthread_cond_init(&state->ready, NULL); + pthread_cond_init(&state->done, NULL); + state->started = 1; + state->handler = display->async_handlers; + state->display = display; + if (pthread_create( + &state->thread, NULL, run_worker, state) != 0) { + return -1; + } + } + } + for (size_t index = 0; index < 2; ++index) { + persistent_worker *state = &concurrent_workers[index]; + + pthread_mutex_lock(&state->lock); + ++state->generation; + pthread_cond_signal(&state->ready); + pthread_mutex_unlock(&state->lock); + } + for (size_t index = 0; index < 2; ++index) { + persistent_worker *state = &concurrent_workers[index]; + + pthread_mutex_lock(&state->lock); + while (state->completed_generation != state->generation) { + pthread_cond_wait(&state->done, &state->lock); + } + pthread_mutex_unlock(&state->lock); + } + return ASYNC_PROBE_EVENTS_RETURN; + } + pthread_mutex_lock(&worker.lock); + if (!worker.started) { + worker.started = 1; + worker.handler = display->async_handlers; + worker.display = display; + if (pthread_create(&worker.thread, NULL, run_worker, + &worker) != 0) { + pthread_mutex_unlock(&worker.lock); + return -1; + } + } + ++worker.generation; + pthread_cond_signal(&worker.ready); + while (worker.completed_generation != worker.generation) { + pthread_cond_wait(&worker.done, &worker.lock); + } + pthread_mutex_unlock(&worker.lock); + return ASYNC_PROBE_EVENTS_RETURN; +} + +int XFlush(Display *display) +{ + (void)display; + pthread_mutex_lock(&worker.lock); + worker.shutdown = 1; + pthread_cond_signal(&worker.ready); + pthread_mutex_unlock(&worker.lock); + if (pthread_join(worker.thread, NULL) != 0) { + return -1; + } + if (concurrent_started) { + for (size_t index = 0; index < 2; ++index) { + persistent_worker *state = &concurrent_workers[index]; + + pthread_mutex_lock(&state->lock); + state->shutdown = 1; + pthread_cond_signal(&state->ready); + pthread_mutex_unlock(&state->lock); + } + for (size_t index = 0; index < 2; ++index) { + if (pthread_join( + concurrent_workers[index].thread, NULL) != 0) { + return -1; + } + } + } + return ASYNC_PROBE_FLUSH_RETURN; +} diff --git a/tests/integration/kzt-tls-dlopen-stress-shared.h b/tests/integration/kzt-tls-dlopen-stress-shared.h new file mode 100644 index 0000000000..3c5df51d30 --- /dev/null +++ b/tests/integration/kzt-tls-dlopen-stress-shared.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef KZT_TLS_DLOPEN_STRESS_SHARED_H +#define KZT_TLS_DLOPEN_STRESS_SHARED_H + +void kzt_tls_stress_mark_fork_child(void); + +#endif diff --git a/tests/integration/kzt-tls-fork-lifecycle-guest.c b/tests/integration/kzt-tls-fork-lifecycle-guest.c new file mode 100644 index 0000000000..fa847f55fc --- /dev/null +++ b/tests/integration/kzt-tls-fork-lifecycle-guest.c @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(c) do { if (!(c)) { fprintf(stderr, "FAIL:%d %s errno=%d\n", __LINE__, #c, errno); exit(1); } } while (0) +static const char *mode; +static int phase, ack[2], data[2], word; +static int *(*cell)(void); +static long worker_tid; +static uintptr_t control[2]; +static void fork_once(void); +static void deny_fork(int reject_errno); +static int idle_stop; + +static void *guest_idle(void *unused) +{ + const struct timespec delay = { .tv_nsec = 1000000 }; + (void)unused; + while (!__atomic_load_n(&idle_stop, __ATOMIC_ACQUIRE)) { + nanosleep(&delay, NULL); + } + return NULL; +} + +static Bool callback(Display *d, xReply *r, char *b, int n, XPointer opaque) +{ + char byte = 1; + (void)d; (void)r; (void)b; (void)n; (void)opaque; + long tid = syscall(SYS_gettid); + if (worker_tid && worker_tid != tid) return 90; + worker_tid = tid; + control[0] = tid; + if (!strncmp(mode, "attached-seccomp", 16)) { + int reject_errno = !strcmp(mode, "attached-seccomp") ? EPERM : 512; + + deny_fork(reject_errno); + for (int i = 0; i < 2; i++) { + errno = 0; + CHECK(fork() == -1 && errno == reject_errno); + } + } else if (!strcmp(mode, "dtv")) { + if (phase == 1) { + if (*cell() != 17) return 91; + *cell() = 12345; + } else if (phase == 2) { + int value = *cell(); + fprintf(stderr, "RETAINED_TLS_VALUE=%d expected=12345\n", value); + if (value != 12345) return 92; + } + } else if (!strcmp(mode, "read") || !strcmp(mode, "futex")) { + if (write(ack[1], &byte, 1) != 1) return 93; + if (!strcmp(mode, "read")) { + if (syscall(SYS_read, data[0], &byte, 1) != 1) return 94; + } else { + while (!__atomic_load_n(&word, __ATOMIC_ACQUIRE)) { + if (syscall(SYS_futex, &word, FUTEX_WAIT_PRIVATE, 0, + NULL, NULL, 0) < 0 && errno != EAGAIN && errno != EINTR) return 95; + } + } + } else if (!strcmp(mode, "flush")) { + if (write(ack[1], &byte, 1) != 1) return 96; + flockfile(stderr); + funlockfile(stderr); + } + return 0; +} + +static void fork_once(void) +{ + int status; + pid_t child = fork(); + CHECK(child >= 0); + if (!child) _exit(0); + CHECK(waitpid(child, &status, 0) == child); + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fprintf(stderr, "CHILD_STATUS=%#x signal=%d\n", status, + WIFSIGNALED(status) ? WTERMSIG(status) : 0); + } + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 0); +} + +static void deny_fork(int reject_errno) +{ + struct sock_filter code[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_clone, 3, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_fork, 2, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_vfork, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | reject_errno), + }; + struct sock_fprog filter = { sizeof(code) / sizeof(code[0]), code }; + CHECK(prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) == 0); + CHECK(prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &filter) == 0); +} + +int main(int argc, char **argv) +{ + Display *display = calloc(1, sizeof(*display)); + _XAsyncHandler handler = { 0 }; + char byte = 1; + pthread_t idle_thread; + CHECK(argc == 4 && display); + mode = argv[1]; + handler.handler = callback; + handler.data = (XPointer)control; + display->async_handlers = &handler; + (void)dlerror(); + if (!strcmp(mode, "dtv")) { + void *keep, *last; + CHECK(XEventsQueued(display, 0) == 0); + for (int i = 0; i < 3; i++) { + void *temporary = dlopen(argv[3], RTLD_NOW | RTLD_LOCAL); + CHECK(temporary && dlclose(temporary) == 0); + } + keep = dlopen(argv[2], RTLD_NOW | RTLD_LOCAL); + CHECK(keep); + *(void **)(&cell) = dlsym(keep, "review_tls_cell"); + CHECK(cell); + phase = 1; + CHECK(XEventsQueued(display, 0) == 0); + for (int i = 0; i < 3; i++) { + void *temporary = dlopen(argv[3], RTLD_NOW | RTLD_LOCAL); + CHECK(temporary && dlclose(temporary) == 0); + } + last = dlopen(argv[3], RTLD_NOW | RTLD_LOCAL); + CHECK(last); + phase = 2; + CHECK(XEventsQueued(display, 0) == 0); + CHECK(XFlush(display) == 0); + CHECK(dlclose(last) == 0 && dlclose(keep) == 0); + } else if (!strncmp(mode, "attached-seccomp", 16)) { + CHECK(XEventsQueued(display, 0) == 0); + CHECK(XFlush(display) == 0); + } else if (!strncmp(mode, "seccomp", 7)) { + int reject_errno = !strcmp(mode, "seccomp") ? EPERM : 512; + + deny_fork(reject_errno); + for (int i = 0; i < 2; i++) { + errno = 0; + CHECK(fork() == -1 && errno == reject_errno); + fprintf(stderr, "FORK_DENIED=%d\n", i + 1); + } + CHECK(XEventsQueued(display, 0) == 0); + CHECK(XFlush(display) == 0); + } else { + CHECK(pipe(ack) == 0 && pipe(data) == 0); + if (!strcmp(mode, "flush")) { + char *single_threaded; + + fork_once(); + CHECK(pthread_create(&idle_thread, NULL, guest_idle, NULL) == 0); + single_threaded = dlsym(RTLD_DEFAULT, "__libc_single_threaded"); + CHECK(!single_threaded || !*single_threaded); + CHECK(XNoOp(display) == 0); + (void)syscall(SYS_getpid); + fork_once(); + fprintf(stderr, "POST_FLUSH_FORK_COMPLETE\n"); + flockfile(stderr); + } + display->fd = !strcmp(mode, "read") ? 1 : 2; + control[1] = !strcmp(mode, "read") ? (uintptr_t)data[0] + : !strcmp(mode, "flush") ? (uintptr_t)stderr->_lock + : (uintptr_t)&word; + CHECK(XEventsQueued(display, 1) == 0); + CHECK(read(ack[0], &byte, 1) == 1); + CHECK(XNoOp(display) == 0); + fork_once(); + if (strcmp(mode, "flush")) { + fprintf(stderr, "FORK_RETURNED_BEFORE_WAKE\n"); + } + if (!strcmp(mode, "flush")) { + funlockfile(stderr); + } else if (!strcmp(mode, "read")) { + CHECK(write(data[1], &byte, 1) == 1); + } else { + __atomic_store_n(&word, 1, __ATOMIC_RELEASE); + CHECK(syscall(SYS_futex, &word, FUTEX_WAKE_PRIVATE, 1, NULL, NULL, 0) >= 0); + } + CHECK(XFlush(display) == 0); + if (!strcmp(mode, "flush")) { + __atomic_store_n(&idle_stop, 1, __ATOMIC_RELEASE); + CHECK(pthread_join(idle_thread, NULL) == 0); + } + } + printf("PASS: review lifecycle %s\n", mode); + free(display); + return 0; +} diff --git a/tests/integration/kzt-tls-fork-lifecycle-plugin.c b/tests/integration/kzt-tls-fork-lifecycle-plugin.c new file mode 100644 index 0000000000..69615496b6 --- /dev/null +++ b/tests/integration/kzt-tls-fork-lifecycle-plugin.c @@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +static __thread int value __attribute__((tls_model("global-dynamic"))) = 17; +int *review_tls_cell(void); +int *review_tls_cell(void) { return &value; } diff --git a/tests/integration/kzt-tls-fork-lifecycle-probe.c b/tests/integration/kzt-tls-fork-lifecycle-probe.c new file mode 100644 index 0000000000..94542a2c8c --- /dev/null +++ b/tests/integration/kzt-tls-fork-lifecycle-probe.c @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +static pthread_t worker; +static Display *target; +static unsigned requested, completed; +static int started, stop, result; + +static void *run(void *unused) +{ + (void)unused; + pthread_mutex_lock(&lock); + while (!stop) { + unsigned task; + xReply reply = { 0 }; + while (!stop && requested == completed) { + pthread_cond_wait(&cond, &lock); + } + if (stop) break; + task = requested; + pthread_mutex_unlock(&lock); + result = target->async_handlers->handler(target, &reply, + (char *)&reply, sizeof(reply), target->async_handlers->data); + pthread_mutex_lock(&lock); + completed = task; + pthread_cond_broadcast(&cond); + } + pthread_mutex_unlock(&lock); + return NULL; +} + +int XEventsQueued(Display *display, int mode) +{ + pthread_mutex_lock(&lock); + if (!started) { + target = display; + if (pthread_create(&worker, NULL, run, NULL)) { + pthread_mutex_unlock(&lock); + return -1; + } + started = 1; + } + requested++; + pthread_cond_broadcast(&cond); + if (!mode) { + while (completed != requested) pthread_cond_wait(&cond, &lock); + } + pthread_mutex_unlock(&lock); + return mode ? 0 : result; +} + +int XFlush(Display *display) +{ + (void)display; + pthread_mutex_lock(&lock); + while (completed != requested) pthread_cond_wait(&cond, &lock); + stop = 1; + pthread_cond_broadcast(&cond); + pthread_mutex_unlock(&lock); + if (pthread_join(worker, NULL)) return -1; + return result; +} + +int XNoOp(Display *display) +{ + if (display->fd) { + const uintptr_t *control = (const uintptr_t *)display->async_handlers->data; + char path[128]; + const struct timespec delay = { .tv_nsec = 1000000 }; + + snprintf(path, sizeof(path), "/proc/self/task/%lu/syscall", + (unsigned long)control[0]); + for (int attempt = 0; attempt < 2000; attempt++) { + FILE *file = fopen(path, "r"); + long nr = -1; + unsigned long argument = 0; + + if (file) { + int fields = fscanf(file, "%ld %lx", &nr, &argument); + fclose(file); + if (fields == 2 && argument == control[1] && + nr == (display->fd == 1 ? SYS_read : SYS_futex)) { + fprintf(stderr, "ATTACHED_BLOCKED tid=%lu syscall=%ld argument=%#lx\n", + (unsigned long)control[0], nr, argument); + return 0; + } + } + nanosleep(&delay, NULL); + } + return -1; + } + void **cpu = dlsym(RTLD_DEFAULT, "thread_cpu"); + void (*flush)(void *) = (void (*)(void *))dlsym(RTLD_DEFAULT, "tb_flush"); + (void)display; + if (!cpu || !*cpu || !flush) return -1; + fprintf(stderr, "REQUEST_FULL_TB_FLUSH\n"); + flush(*cpu); + return 0; +} diff --git a/tests/integration/registrations/x11-kzt/meson.build b/tests/integration/registrations/x11-kzt/meson.build index 313827c700..27d4ab66c7 100644 --- a/tests/integration/registrations/x11-kzt/meson.build +++ b/tests/integration/registrations/x11-kzt/meson.build @@ -23,4 +23,87 @@ if 'x86_64-linux-user' in target_dirs and \ ], 'timeout': 120, }] + + + latx_integration_tests += [{ + 'name': 'test-kzt-guest-tls-opt-in', + 'runner': find_program('../../test-kzt-guest-tls-opt-in.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../kzt-guest-tls-opt-in.c'), + files('../../x11-async-bridge-dummy.c'), + files('../../kzt-guest-tls-opt-in-plugin.c'), + ], + 'timeout': 120, + }] + + latx_integration_tests += [{ + 'name': 'test-kzt-pthread-tsd-alias', + 'runner': find_program('../../test-kzt-pthread-tsd-alias.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../kzt-pthread-tsd-alias-guest.c'), + ], + 'timeout': 120, + }] + + latx_integration_tests += [{ + 'name': 'test-kzt-attached-robust', + 'runner': find_program('../../test-kzt-attached-robust.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../kzt-attached-robust-guest.c'), + files('../../kzt-attached-robust-probe.c'), + files('../../x11-async-bridge-dummy.c'), + ], + 'timeout': 120, + }] + + latx_integration_tests += [{ + 'name': 'test-kzt-cxx-tls-lifetime', + 'runner': find_program('../../test-kzt-cxx-tls-lifetime.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../kzt-cxx-tls-lifetime-guest.c'), + files('../../kzt-cxx-tls-lifetime-plugin.cpp'), + files('../../kzt-cxx-tls-lifetime-destructor.cpp'), + files('../../kzt-cxx-tls-lifetime-probe.c'), + files('../../x11-async-bridge-dummy.c'), + ], + 'timeout': 120, + }] + + latx_integration_tests += [{ + 'name': 'test-kzt-tls-dlopen-stress', + 'runner': find_program('../../test-kzt-tls-dlopen-stress.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../kzt-tls-dlopen-stress-guest.c'), + files('../../kzt-host-thread-tls-guest-plugin.c'), + files('../../kzt-host-thread-tls-guest-plugin-b.c'), + files('../../kzt-tls-dlopen-stress-ie-plugin.c'), + files('../../kzt-tls-dlopen-stress-probe.c'), + files('../../x11-async-bridge-dummy.c'), + ], + 'timeout': 180, + }] + + + + foreach mode : ['dtv', 'read', 'futex', 'seccomp', 'seccomp-restart-code', + 'attached-seccomp', + 'attached-seccomp-restart-code', 'flush'] + latx_integration_tests += [{ + 'name': 'test-kzt-tls-fork-' + mode, + 'runner': find_program('../../test-kzt-tls-fork-lifecycle.sh'), + 'args': [ + emulators['latx-x86_64'], mode, + files('../../kzt-tls-fork-lifecycle-guest.c'), + files('../../kzt-tls-fork-lifecycle-probe.c'), + files('../../kzt-tls-fork-lifecycle-plugin.c'), + files('../../x11-async-bridge-dummy.c'), + ], + 'timeout': 60, + }] + endforeach endif diff --git a/tests/integration/test-kzt-attached-robust.sh b/tests/integration/test-kzt-attached-robust.sh new file mode 100755 index 0000000000..f3ce205968 --- /dev/null +++ b/tests/integration/test-kzt-attached-robust.sh @@ -0,0 +1,99 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later + +set -eu +export LATX_KZT_GUEST_TLS=1 + +emulator=$1 +guest_source=$2 +probe_source=$3 +dummy_source=$4 +source_dir=$(dirname "$guest_source") + +if [ "$(uname -m)" != loongarch64 ]; then + echo "SKIP: KZT attached robust mutex test requires a LoongArch host" + exit 77 +fi + +guest_root=${LATX_X86_64_SYSROOT:-/usr/gnemul/latx-x86_64} +guest_compiler=${LATX_X86_64_CC:-x86_64-linux-gnu-gcc} +native_compiler=${LATX_NATIVE_CC:-cc} +guest_artifacts=${LATX_KZT_ROBUST_GUEST_ARTIFACT_DIR:-} +workdir=$(mktemp -d) +guest_lib="$workdir/guest-lib" +guest_program="$workdir/kzt-attached-robust-guest" +host_probe="$workdir/libX11.so.6" + +cleanup() +{ + rm -rf "$workdir" +} +trap cleanup EXIT HUP INT TERM +mkdir -p "$guest_lib" + +"$native_compiler" -shared -fPIC -O2 -Wall -Wextra -Werror \ + -I"$source_dir" -Wl,-soname,libX11.so.6 \ + "$probe_source" -pthread -o "$host_probe" + +if [ -n "$guest_artifacts" ]; then + if [ ! -x "$guest_artifacts/kzt-attached-robust-guest" ] || + [ ! -f "$guest_artifacts/libX11.so.6" ] || + [ ! -f "$guest_artifacts/sources.sha256" ] || + [ ! -f "$guest_artifacts/artifacts.sha256" ]; then + echo "FAIL: incomplete prebuilt attached robust artifacts" >&2 + exit 2 + fi + for source_file in "$guest_source" "$dummy_source"; do + source_name=$(basename "$source_file") + expected_hash=$(sha256sum "$source_file" | awk '{print $1}') + artifact_hash=$(awk -v name="$source_name" \ + '$2 == name {print $1}' \ + "$guest_artifacts/sources.sha256") + + if [ -z "$artifact_hash" ] || + [ "$artifact_hash" != "$expected_hash" ]; then + echo "FAIL: prebuilt robust source mismatch: $source_name" >&2 + exit 2 + fi + done + if ! (cd "$guest_artifacts" && sha256sum -c artifacts.sha256); then + echo "FAIL: prebuilt attached robust artifact mismatch" >&2 + exit 2 + fi + cp "$guest_artifacts/kzt-attached-robust-guest" "$guest_program" + cp "$guest_artifacts/libX11.so.6" "$guest_lib/libX11.so.6" +else + if ! command -v "$guest_compiler" >/dev/null 2>&1; then + echo "SKIP: x86_64 Guest compiler is unavailable" + exit 77 + fi + "$guest_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror -I"$source_dir" \ + -Wl,-soname,libX11.so.6 "$dummy_source" \ + -o "$guest_lib/libX11.so.6" + ln -s libX11.so.6 "$guest_lib/libX11.so" + "$guest_compiler" --sysroot="$guest_root" -O2 \ + -Wall -Wextra -Werror -I"$source_dir" "$guest_source" \ + -L"$guest_lib" -lX11 -ldl -pthread -o "$guest_program" +fi + +for aot_mode in 0 1; do + output="$workdir/output-$aot_mode.log" + set +e + LD_PRELOAD="$host_probe" \ + BOX64_LD_LIBRARY_PATH="$guest_lib" \ + LATX_AOT=$aot_mode LATX_KZT=1 \ + "$emulator" -U LD_PRELOAD \ + -E "LD_LIBRARY_PATH=$guest_lib" -L "$guest_root" \ + "$guest_program" >"$output" 2>&1 + result=$? + set -e + if [ "$result" -ne 0 ]; then + sed -n '1,160p' "$output" >&2 + exit "$result" + fi + grep -q \ + '^PASS: attached robust owner death woke blocked Guest waiter$' \ + "$output" + cat "$output" +done diff --git a/tests/integration/test-kzt-cxx-tls-lifetime.sh b/tests/integration/test-kzt-cxx-tls-lifetime.sh new file mode 100755 index 0000000000..b0e3bfb613 --- /dev/null +++ b/tests/integration/test-kzt-cxx-tls-lifetime.sh @@ -0,0 +1,138 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later + +set -eu +export LATX_KZT_GUEST_TLS=1 + +emulator=$1 +guest_source=$2 +plugin_source=$3 +destructor_source=$4 +probe_source=$5 +dummy_source=$6 +source_dir=$(dirname "$guest_source") + +if [ "$(uname -m)" != loongarch64 ]; then + echo "SKIP: KZT C++ TLS lifetime test requires a LoongArch host" + exit 77 +fi + +guest_root=${LATX_X86_64_SYSROOT:-/usr/gnemul/latx-x86_64} +guest_compiler=${LATX_X86_64_CC:-x86_64-linux-gnu-gcc} +guest_cxx_compiler=${LATX_X86_64_CXX:-x86_64-linux-gnu-g++} +native_compiler=${LATX_NATIVE_CC:-cc} +guest_artifacts=${LATX_KZT_CXX_TLS_GUEST_ARTIFACT_DIR:-} +workdir=$(mktemp -d) +guest_lib="$workdir/guest-lib" +guest_program="$workdir/kzt-cxx-tls-lifetime-guest" +guest_plugin="$guest_lib/libkzt-cxx-tls-lifetime-plugin.so" +guest_destructor="$guest_lib/libkzt-cxx-tls-lifetime-destructor.so" +host_probe="$workdir/libX11.so.6" +output="$workdir/output.log" + +verify_prebuilt_source() +{ + source_file=$1 + source_name=$(basename "$source_file") + expected_hash=$(sha256sum "$source_file" | awk '{print $1}') + artifact_hash=$(awk -v name="$source_name" \ + '$2 == name {print $1}' "$guest_artifacts/sources.sha256") + + if [ -z "$artifact_hash" ] || + [ "$artifact_hash" != "$expected_hash" ]; then + echo "FAIL: prebuilt C++ TLS source mismatch: $source_name" >&2 + exit 2 + fi +} + +cleanup() +{ + rm -rf "$workdir" +} +trap cleanup EXIT HUP INT TERM + +if [ ! -d "$guest_root" ]; then + echo "SKIP: x86_64 Guest sysroot not found: $guest_root" + exit 77 +fi +if ! command -v "$native_compiler" >/dev/null 2>&1; then + echo "SKIP: native compiler not found: $native_compiler" + exit 77 +fi +mkdir -p "$guest_lib" + +"$native_compiler" -shared -fPIC -O2 -Wall -Wextra -Werror \ + -I"$source_dir" -Wl,-soname,libX11.so.6 \ + "$probe_source" -pthread -o "$host_probe" + +if [ -n "$guest_artifacts" ]; then + if [ ! -x "$guest_artifacts/kzt-cxx-tls-lifetime-guest" ] || + [ ! -f "$guest_artifacts/libX11.so.6" ] || + [ ! -f \ + "$guest_artifacts/libkzt-cxx-tls-lifetime-plugin.so" ] || + [ ! -f \ + "$guest_artifacts/libkzt-cxx-tls-lifetime-destructor.so" ] || + [ ! -f "$guest_artifacts/sources.sha256" ] || + [ ! -f "$guest_artifacts/artifacts.sha256" ]; then + echo "FAIL: incomplete prebuilt C++ TLS lifetime artifacts" >&2 + exit 2 + fi + verify_prebuilt_source "$guest_source" + verify_prebuilt_source "$plugin_source" + verify_prebuilt_source "$destructor_source" + verify_prebuilt_source \ + "$source_dir/kzt-cxx-tls-lifetime-shared.h" + verify_prebuilt_source "$dummy_source" + if ! (cd "$guest_artifacts" && sha256sum -c artifacts.sha256); then + echo "FAIL: prebuilt C++ TLS artifact hash mismatch" >&2 + exit 2 + fi + cp "$guest_artifacts/kzt-cxx-tls-lifetime-guest" "$guest_program" + cp "$guest_artifacts/libX11.so.6" "$guest_lib/libX11.so.6" + cp "$guest_artifacts/libkzt-cxx-tls-lifetime-plugin.so" \ + "$guest_plugin" + cp "$guest_artifacts/libkzt-cxx-tls-lifetime-destructor.so" \ + "$guest_destructor" +else + if ! command -v "$guest_compiler" >/dev/null 2>&1 || + ! command -v "$guest_cxx_compiler" >/dev/null 2>&1; then + echo "SKIP: x86_64 Guest compilers are unavailable" + exit 77 + fi + "$guest_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror -I"$source_dir" \ + -Wl,-soname,libX11.so.6 "$dummy_source" \ + -o "$guest_lib/libX11.so.6" + "$guest_cxx_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror -I"$source_dir" \ + -Wl,-soname,libkzt-cxx-tls-lifetime-destructor.so \ + "$destructor_source" -o "$guest_destructor" + "$guest_cxx_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror -I"$source_dir" "$plugin_source" \ + -L"$guest_lib" -lkzt-cxx-tls-lifetime-destructor \ + -Wl,-rpath,'$ORIGIN' -o "$guest_plugin" + "$guest_compiler" --sysroot="$guest_root" -O2 \ + -Wall -Wextra -Werror -I"$source_dir" "$guest_source" \ + -L"$guest_lib" -lX11 -ldl -pthread -o "$guest_program" +fi + +for aot_mode in 0 1; do + output="$workdir/output-aot-$aot_mode.log" + set +e + LD_PRELOAD="$host_probe" \ + BOX64_LD_LIBRARY_PATH="$guest_lib" \ + LATX_AOT=$aot_mode LATX_KZT=1 \ + "$emulator" -U LD_PRELOAD \ + -E "LD_LIBRARY_PATH=$guest_lib" -L "$guest_root" \ + "$guest_program" "$guest_plugin" >"$output" 2>&1 + result=$? + set -e + if [ "$result" -ne 0 ]; then + sed -n '1,200p' "$output" >&2 + echo "FAIL: LATX_AOT=$aot_mode exited $result" >&2 + exit "$result" + fi + grep -q '^PASS: C++ thread_local DSO survived until Host-thread exit$' \ + "$output" + cat "$output" +done diff --git a/tests/integration/test-kzt-guest-tls-opt-in.sh b/tests/integration/test-kzt-guest-tls-opt-in.sh new file mode 100755 index 0000000000..fb825c88af --- /dev/null +++ b/tests/integration/test-kzt-guest-tls-opt-in.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later +set -eu + +emulator=$1 +source_file=$2 +dummy_source=$3 +plugin_source=$4 +guest_root=${LATX_X86_64_SYSROOT:-/usr/gnemul/latx-x86_64} +guest_cc=${LATX_X86_64_CC:-x86_64-linux-gnu-gcc} +native_cc=${LATX_NATIVE_CC:-cc} + +if [ "$(uname -m)" != loongarch64 ] || + ! command -v "$guest_cc" >/dev/null 2>&1 || + ! command -v "$native_cc" >/dev/null 2>&1; then + echo "SKIP: requires LoongArch and Guest/Host C compilers" + exit 77 +fi +task_dir=$(mktemp -d) +trap 'rm -rf "$task_dir"' EXIT HUP INT TERM +mkdir -p "$task_dir/guest" +"$native_cc" -shared -fPIC -O2 -Wall -Wextra -Werror -DHOST_PROBE \ + "$source_file" -pthread -Wl,-soname,libX11.so.6 \ + -o "$task_dir/libX11.so.6" +"$guest_cc" --sysroot="$guest_root" -shared -fPIC \ + -I"$(dirname "$dummy_source")" "$dummy_source" \ + -Wl,-soname,libX11.so.6 -o "$task_dir/guest/libX11.so.6" +"$guest_cc" --sysroot="$guest_root" -O2 -Wall -Wextra -Werror \ + "$source_file" -L"$task_dir/guest" -l:libX11.so.6 \ + -Wl,--no-as-needed -ldl -Wl,--as-needed -pthread \ + -o "$task_dir/guest/opt-in" +"$guest_cc" --sysroot="$guest_root" -shared -fPIC -O2 \ + "$plugin_source" -o "$task_dir/guest/late.so" + +for setting in unset 0 1; do + mode=existing + if [ "$setting" = 1 ]; then mode=attached; fi + if [ "$setting" = unset ]; then + unset LATX_KZT_GUEST_TLS + else + export LATX_KZT_GUEST_TLS=$setting + fi + LD_PRELOAD="$task_dir/libX11.so.6" \ + BOX64_LD_LIBRARY_PATH="$task_dir/guest" \ + LATX_AOT=0 LATX_KZT=1 LATX_KZT_LIBS=core,x11 \ + "$emulator" -U LD_PRELOAD \ + -E "LD_LIBRARY_PATH=$task_dir/guest" \ + -L "$guest_root" "$task_dir/guest/opt-in" "$mode" "$task_dir/guest/late.so" + if [ "$setting" = 1 ] && [ -n "${LATX_KZT_BOOTSTRAP_GDB:-}" ]; then + for check in kzt-bootstrap-tid.gdb kzt-attach-busy.gdb; do + BOX64_LD_LIBRARY_PATH="$task_dir/guest" \ + LATX_AOT=0 LATX_KZT=1 LATX_KZT_LIBS=core,x11 \ + "$LATX_KZT_BOOTSTRAP_GDB" -q -batch \ + -ex "set environment LD_PRELOAD=$task_dir/libX11.so.6" \ + -x "$(dirname "$source_file")/$check" \ + --args "$emulator" -U LD_PRELOAD \ + -E "LD_LIBRARY_PATH=$task_dir/guest" \ + -L "$guest_root" "$task_dir/guest/opt-in" \ + attached "$task_dir/guest/late.so" + done + fi +done diff --git a/tests/integration/test-kzt-pthread-tsd-alias.sh b/tests/integration/test-kzt-pthread-tsd-alias.sh new file mode 100755 index 0000000000..f57ad50913 --- /dev/null +++ b/tests/integration/test-kzt-pthread-tsd-alias.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later + +set -eu + +emulator=$1 +guest_source=$2 +source_dir=$(dirname "$guest_source") + +if [ "$(uname -m)" != loongarch64 ]; then + echo "SKIP: KZT pthread TSD alias test requires a LoongArch host" + exit 77 +fi + +guest_root=${LATX_X86_64_SYSROOT:-/usr/gnemul/latx-x86_64} +guest_compiler=${LATX_X86_64_CC:-x86_64-linux-gnu-gcc} +guest_artifacts=${LATX_KZT_TSD_GUEST_ARTIFACT_DIR:-} +workdir=$(mktemp -d) +guest_program="$workdir/kzt-pthread-tsd-alias-guest" + +cleanup() +{ + rm -rf "$workdir" +} +trap cleanup EXIT HUP INT TERM + +if [ ! -d "$guest_root" ]; then + echo "SKIP: x86_64 Guest sysroot not found: $guest_root" + exit 77 +fi + +if [ -n "$guest_artifacts" ]; then + if [ ! -x "$guest_artifacts/kzt-pthread-tsd-alias-guest" ] || + [ ! -f "$guest_artifacts/sources.sha256" ]; then + echo "FAIL: incomplete prebuilt Guest TSD artifacts" >&2 + exit 2 + fi + for source_file in \ + "$guest_source" \ + "$source_dir/kzt-pthread-tsd-alias.h"; do + source_name=$(basename "$source_file") + expected_source_hash=$(sha256sum "$source_file" | awk '{print $1}') + artifact_source_hash=$(awk -v name="$source_name" \ + '$2 == name {print $1}' "$guest_artifacts/sources.sha256") + if [ -z "$artifact_source_hash" ] || + [ "$artifact_source_hash" != "$expected_source_hash" ]; then + echo "FAIL: prebuilt Guest TSD source mismatch: " \ + "$source_name" >&2 + exit 2 + fi + done + cp "$guest_artifacts/kzt-pthread-tsd-alias-guest" "$guest_program" +else + if ! command -v "$guest_compiler" >/dev/null 2>&1; then + echo "SKIP: x86_64 cross compiler not found: $guest_compiler" + exit 77 + fi + "$guest_compiler" --sysroot="$guest_root" -O2 \ + -Wall -Wextra -Werror "$guest_source" -ldl -pthread \ + -o "$guest_program" +fi + +for aot_mode in 0 1; do + for kzt_mode in 0 1; do + output="$workdir/output-$aot_mode-$kzt_mode.log" + guest_args= + if [ "$kzt_mode" -eq 0 ]; then + guest_args=--single-thread + fi + set +e + LATX_AOT=$aot_mode LATX_KZT=$kzt_mode \ + "$emulator" -L "$guest_root" "$guest_program" $guest_args \ + >"$output" 2>&1 + result=$? + set -e + if [ "$result" -ne 0 ]; then + sed -n '1,160p' "$output" >&2 + echo "FAIL: LATX_AOT=$aot_mode LATX_KZT=$kzt_mode " \ + "exited $result" >&2 + exit "$result" + fi + grep -q '^PASS: Guest pthread TSD remains Guest-libc authoritative$' \ + "$output" + done +done + +echo "PASS: KZT on/off preserve Guest pthread TSD alias coherence" diff --git a/tests/integration/test-kzt-tls-dlopen-stress.sh b/tests/integration/test-kzt-tls-dlopen-stress.sh new file mode 100755 index 0000000000..fc63556269 --- /dev/null +++ b/tests/integration/test-kzt-tls-dlopen-stress.sh @@ -0,0 +1,149 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later + +set -eu +export LATX_KZT_GUEST_TLS=1 + +emulator=$1 +guest_source=$2 +plugin_a_source=$3 +plugin_b_source=$4 +ie_plugin_source=$5 +probe_source=$6 +dummy_source=$7 +source_dir=$(dirname "$guest_source") + +if [ "$(uname -m)" != loongarch64 ]; then + echo "SKIP: KZT TLS dlopen stress requires a LoongArch host" + exit 77 +fi + +guest_root=${LATX_X86_64_SYSROOT:-/usr/gnemul/latx-x86_64} +guest_compiler=${LATX_X86_64_CC:-x86_64-linux-gnu-gcc} +native_compiler=${LATX_NATIVE_CC:-cc} +guest_artifacts=${LATX_KZT_TLS_STRESS_ARTIFACT_DIR:-} +workdir=$(mktemp -d) +guest_lib="$workdir/guest-lib" +guest_program="$workdir/kzt-tls-dlopen-stress-guest" +plugin_a="$guest_lib/libkzt-tls-stress-a.so" +plugin_b="$guest_lib/libkzt-tls-stress-b.so" +ie_plugin="$guest_lib/libkzt-tls-stress-ie.so" +host_probe="$workdir/libX11.so.6" +wrapped_probe="$guest_lib/libxcb.so.1" + +verify_prebuilt_source() +{ + source_file=$1 + source_name=$(basename "$source_file") + expected_hash=$(sha256sum "$source_file" | awk '{print $1}') + artifact_hash=$(awk -v name="$source_name" \ + '$2 == name {print $1}' "$guest_artifacts/sources.sha256") + + if [ -z "$artifact_hash" ] || + [ "$artifact_hash" != "$expected_hash" ]; then + echo "FAIL: prebuilt TLS stress source mismatch: $source_name" >&2 + exit 2 + fi +} + +cleanup() +{ + rm -rf "$workdir" +} +trap cleanup EXIT HUP INT TERM +mkdir -p "$guest_lib" + +"$native_compiler" -shared -fPIC -O2 -Wall -Wextra -Werror \ + -I"$source_dir" -Wl,-soname,libX11.so.6 \ + "$probe_source" -pthread -o "$host_probe" + +if [ -n "$guest_artifacts" ]; then + if [ ! -x "$guest_artifacts/kzt-tls-dlopen-stress-guest" ] || + [ ! -f "$guest_artifacts/libX11.so.6" ] || + [ ! -f "$guest_artifacts/libxcb.so.1" ] || + [ ! -f "$guest_artifacts/libkzt-tls-stress-a.so" ] || + [ ! -f "$guest_artifacts/libkzt-tls-stress-b.so" ] || + [ ! -f "$guest_artifacts/libkzt-tls-stress-ie.so" ] || + [ ! -f "$guest_artifacts/sources.sha256" ] || + [ ! -f "$guest_artifacts/artifacts.sha256" ]; then + echo "FAIL: incomplete prebuilt TLS stress artifacts" >&2 + exit 2 + fi + verify_prebuilt_source "$guest_source" + verify_prebuilt_source "$plugin_a_source" + verify_prebuilt_source "$plugin_b_source" + verify_prebuilt_source "$ie_plugin_source" + verify_prebuilt_source "$dummy_source" + verify_prebuilt_source \ + "$source_dir/kzt-tls-dlopen-stress-shared.h" + if ! (cd "$guest_artifacts" && sha256sum -c artifacts.sha256); then + echo "FAIL: prebuilt TLS stress artifact hash mismatch" >&2 + exit 2 + fi + cp "$guest_artifacts/kzt-tls-dlopen-stress-guest" "$guest_program" + cp "$guest_artifacts/libX11.so.6" "$guest_lib/libX11.so.6" + cp "$guest_artifacts/libxcb.so.1" "$wrapped_probe" + cp "$guest_artifacts/libkzt-tls-stress-a.so" "$plugin_a" + cp "$guest_artifacts/libkzt-tls-stress-b.so" "$plugin_b" + cp "$guest_artifacts/libkzt-tls-stress-ie.so" "$ie_plugin" +else + if ! command -v "$guest_compiler" >/dev/null 2>&1; then + exit 77 + fi + "$guest_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror -I"$source_dir" \ + -Wl,-soname,libX11.so.6 "$dummy_source" \ + -o "$guest_lib/libX11.so.6" + "$guest_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror -I"$source_dir" \ + -Wl,-soname,libxcb.so.1 "$dummy_source" \ + -o "$wrapped_probe" + "$guest_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror "$plugin_a_source" -o "$plugin_a" + "$guest_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror "$plugin_b_source" -o "$plugin_b" + "$guest_compiler" --sysroot="$guest_root" -shared -fPIC -O2 \ + -Wall -Wextra -Werror -I"$source_dir" \ + -ftls-model=initial-exec \ + "$ie_plugin_source" -o "$ie_plugin" + "$guest_compiler" --sysroot="$guest_root" -O2 \ + -Wall -Wextra -Werror -I"$source_dir" "$guest_source" \ + -Wl,--export-dynamic -L"$guest_lib" -lX11 -ldl -pthread \ + -o "$guest_program" +fi + +for aot_mode in 0 1; do + aot_home="$workdir/aot-home-$aot_mode" + cache_passes=cold + + mkdir -p "$aot_home" + if [ "$aot_mode" -eq 1 ]; then + cache_passes='cold warm' + fi + for cache_pass in $cache_passes; do + output="$workdir/output-$aot_mode-$cache_pass.log" + set +e + HOME="$aot_home" LD_PRELOAD="$host_probe" \ + BOX64_LD_LIBRARY_PATH="$guest_lib" \ + LATX_AOT=$aot_mode LATX_KZT=1 \ + "$emulator" -U LD_PRELOAD \ + -E "LD_LIBRARY_PATH=$guest_lib" -L "$guest_root" \ + "$guest_program" "$plugin_a" "$plugin_b" "$ie_plugin" \ + >"$output" 2>&1 + result=$? + set -e + if [ "$result" -ne 0 ]; then + sed -n '1,200p' "$output" >&2 + exit "$result" + fi + grep -q '^PASS: attached Guest TLS survived 64 A/B dlopen cycles ' \ + "$output" + if [ "$aot_mode" -eq 1 ] && [ "$cache_pass" = cold ] && + ! find "$aot_home/.cache/latx" -type f \ + -name '*.aot2' -size +0c -print -quit | grep -q .; then + echo "FAIL: cold AOT run did not publish a cache" >&2 + exit 1 + fi + cat "$output" + done +done diff --git a/tests/integration/test-kzt-tls-fork-lifecycle.sh b/tests/integration/test-kzt-tls-fork-lifecycle.sh new file mode 100755 index 0000000000..e67656005d --- /dev/null +++ b/tests/integration/test-kzt-tls-fork-lifecycle.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later +set -eu +emulator=$1 +mode=$2 +guest_source=$3 +probe_source=$4 +plugin_source=$5 +dummy_source=$6 +guest_root=${LATX_X86_64_SYSROOT:-/usr/gnemul/latx-x86_64} +guest_cc=${LATX_X86_64_CC:-x86_64-linux-gnu-gcc} +native_cc=${LATX_NATIVE_CC:-cc} +x11_include=${LATX_X11_INCLUDE:-/usr/include} +artifacts=${LATX_KZT_FORK_GUEST_ARTIFACT_DIR:-} +if [ "$(uname -m)" != loongarch64 ] || [ ! -d "$guest_root" ]; then + echo 'SKIP: requires LoongArch and an x86-64 Guest runtime' + exit 77 +fi +if [ "$mode" = flush ] && ! command -v gdb >/dev/null 2>&1; then + echo 'SKIP: gdb is required to observe hook execution after full TB flush' + exit 77 +fi +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +mkdir "$work/guest-lib" +"$native_cc" -shared -fPIC -O2 -Wall -Wextra -Werror -I"$x11_include" \ + "$probe_source" -pthread -ldl -Wl,-soname,libX11.so.6 -o "$work/libX11.so.6" +if [ -n "$artifacts" ]; then + for source in "$guest_source" "$plugin_source" "$dummy_source"; do + name=$(basename "$source") + actual=$(sha256sum "$source" | awk '{print $1}') + expected=$(awk -v name="$name" '$2 == name {print $1}' "$artifacts/sources.sha256") + [ "$actual" = "$expected" ] || { echo "FAIL: source mismatch $name"; exit 1; } + done + (cd "$artifacts" && sha256sum -c artifacts.sha256) + cp "$artifacts/guest" "$work/guest" + cp "$artifacts/guest-lib/"*.so* "$work/guest-lib/" +else + if ! command -v "$guest_cc" >/dev/null 2>&1; then + echo 'SKIP: x86-64 compiler unavailable; provide verified Guest artifacts' + exit 77 + fi + for name in a b; do + "$guest_cc" --sysroot="$guest_root" -shared -fPIC -O2 "$plugin_source" \ + -o "$work/guest-lib/libtls-$name.so" + done + "$guest_cc" --sysroot="$guest_root" -shared -fPIC -O2 -I"$x11_include" \ + "$dummy_source" -Wl,-soname,libX11.so.6 -o "$work/guest-lib/libX11.so.6" + "$guest_cc" --sysroot="$guest_root" -fPIE -pie -O2 -I"$x11_include" \ + "$guest_source" -L"$work/guest-lib" -Wl,--no-as-needed -l:libX11.so.6 \ + -ldl -pthread -o "$work/guest" +fi +export LATX_KZT=1 LATX_KZT_GUEST_TLS=1 LATX_KZT_LIBS=x11 LATX_AOT=0 +export BOX64_LD_LIBRARY_PATH="$work/guest-lib" +set -- "$emulator" -U LD_PRELOAD -E "LD_LIBRARY_PATH=$work/guest-lib" \ + -L "$guest_root" "$work/guest" "$mode" \ + "$work/guest-lib/libtls-a.so" "$work/guest-lib/libtls-b.so" +if [ "$mode" = flush ]; then + cat > "$work/hooks.gdb" <<'GDB' +set pagination off +set confirm off +set breakpoint pending on +set $hooks_after_flush = 0 +set $flushes = 0 +break do_tb_flush +commands +silent +set $flushes = $flushes + 1 +continue +end +break kzt_guest_tls_fork_prepare_early +commands +silent +if $flushes > 0 +set $hooks_after_flush = $hooks_after_flush + 1 +end +continue +end +run +printf "POST_FLUSH_HOOKS=%d FLUSHES=%d\n", $hooks_after_flush, $flushes +quit +GDB + LD_PRELOAD="$work/libX11.so.6" gdb -q -batch -x "$work/hooks.gdb" \ + --args "$@" > "$work/output.log" 2>&1 + grep -Eq '^POST_FLUSH_HOOKS=[2-9][0-9]* FLUSHES=[1-9][0-9]*$' "$work/output.log" || { + cat "$work/output.log"; exit 1; + } +else + LD_PRELOAD="$work/libX11.so.6" "$@" > "$work/output.log" 2>&1 || { + cat "$work/output.log"; exit 1; + } +fi +cat "$work/output.log" +grep -q "^PASS: review lifecycle $mode$" "$work/output.log" diff --git a/tests/integration/x11-async-bridge-dummy.c b/tests/integration/x11-async-bridge-dummy.c index dac7cc7cc1..341e5dc70d 100644 --- a/tests/integration/x11-async-bridge-dummy.c +++ b/tests/integration/x11-async-bridge-dummy.c @@ -18,3 +18,19 @@ int XFlush(Display *display) fputs("ASYNC_GUEST_DUMMY_CALLED:XFlush\n", stderr); return -1; } + +int (*XSetAfterFunction(Display *display, + int (*callback)(Display *)))(Display *) +{ + (void)display; + (void)callback; + fputs("ASYNC_GUEST_DUMMY_CALLED:XSetAfterFunction\n", stderr); + return NULL; +} + +int XNoOp(Display *display) +{ + (void)display; + fputs("ASYNC_GUEST_DUMMY_CALLED:XNoOp\n", stderr); + return -1; +} diff --git a/tests/unit/kzt/check_kzt_tls_refresh_boundary.py b/tests/unit/kzt/check_kzt_tls_refresh_boundary.py new file mode 100644 index 0000000000..87099a1ee6 --- /dev/null +++ b/tests/unit/kzt/check_kzt_tls_refresh_boundary.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later + +import pathlib +import sys + + +def fail(message: str) -> None: + raise SystemExit(f"FAIL: {message}") + + +callback = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +wrappedlibdl = pathlib.Path(sys.argv[2]).read_text(encoding="utf-8") +myalign = pathlib.Path(sys.argv[3]).read_text(encoding="utf-8") + +if callback.count("kzt_guest_tls_refresh_if_needed(") != 1: + fail("callback entries must share the generation-gated scope") +if callback.count("callback_scope_enter(&scope,") != 4: + fail("all callback entries, including float results, must enter the scope") +if "kzt_guest_tls_refresh(" in callback: + fail("callback hot paths must not invoke unconditional TLS refresh") +if wrappedlibdl.count("kzt_guest_tls_refresh(cpu)") < 3: + fail("Guest loader mutation paths must retain unconditional refresh") + +loader_callback = myalign[myalign.index( + "static void kzt_dynamic_library_change_callback") :] +begin = loader_callback.index("kzt_guest_tls_loader_event_begin()") +observe = loader_callback.index("kzt_public_loader_observer_refresh(") +if begin > observe: + fail("loader transition must become dirty before observer refresh") + +print("kzt Guest TLS refresh boundary checks: PASS") diff --git a/tests/unit/kzt/test_kzt_guest_tls_epoch.c b/tests/unit/kzt/test_kzt_guest_tls_epoch.c new file mode 100644 index 0000000000..76016b327c --- /dev/null +++ b/tests/unit/kzt/test_kzt_guest_tls_epoch.c @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include + +#include "kzt-guest-tls-epoch.h" + +#define CHECK(condition) \ + do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, \ + __LINE__, #condition); \ + exit(1); \ + } \ + } while (0) + +static void test_only_stable_matching_epoch_can_skip_refresh(void) +{ + CHECK(kzt_guest_tls_epoch_can_reuse( + 4, 4, 1, 0, 7, 7)); + CHECK(!kzt_guest_tls_epoch_can_reuse( + 3, 3, 1, 0, 7, 7)); + CHECK(!kzt_guest_tls_epoch_can_reuse( + 4, 6, 1, 0, 7, 7)); + CHECK(!kzt_guest_tls_epoch_can_reuse( + 4, 4, 0, 0, 7, 7)); + CHECK(!kzt_guest_tls_epoch_can_reuse( + 4, 4, 1, 8, 7, 7)); + CHECK(!kzt_guest_tls_epoch_can_reuse( + 4, 4, 1, 0, 6, 7)); +} + +static void test_loader_events_publish_only_complete_epochs(void) +{ + CHECK(kzt_guest_tls_epoch_begin_change(0) == 0); + CHECK(kzt_guest_tls_epoch_begin_change(2) == 3); + CHECK(kzt_guest_tls_epoch_begin_change(3) == 3); + CHECK(kzt_guest_tls_epoch_publish_stable(3) == 4); + CHECK(kzt_guest_tls_epoch_publish_stable(4) == 4); + CHECK(kzt_guest_tls_epoch_begin_change(UINT32_MAX - 1) == UINT32_MAX); + CHECK(kzt_guest_tls_epoch_publish_stable(UINT32_MAX) == UINT32_MAX); +} + +static void test_fork_requires_revalidation_before_fast_path(void) +{ + CHECK(kzt_guest_tls_epoch_after_fork(0) == 0); + CHECK(kzt_guest_tls_epoch_after_fork(3) == 1); + CHECK(kzt_guest_tls_epoch_after_fork(4) == 1); +} + +int main(void) +{ + test_only_stable_matching_epoch_can_skip_refresh(); + test_loader_events_publish_only_complete_epochs(); + test_fork_requires_revalidation_before_fast_path(); + puts("kzt Guest TLS epoch tests: PASS"); + return 0; +} diff --git a/tests/unit/kzt/test_kzt_guest_tls_policy.c b/tests/unit/kzt/test_kzt_guest_tls_policy.c new file mode 100644 index 0000000000..203b038a10 --- /dev/null +++ b/tests/unit/kzt/test_kzt_guest_tls_policy.c @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#include +#include +#include "kzt-runtime.h" + +int option_kzt; +int option_kzt_guest_tls; +uint32_t kzt_effective_groups; + +static void check(int expected) +{ + if (latx_kzt_guest_tls_enabled() != expected) { + fprintf(stderr, "TLS policy mismatch: kzt=%d tls=%d groups=%u\n", + option_kzt, option_kzt_guest_tls, kzt_effective_groups); + exit(1); + } +} + +int main(void) +{ + check(0); + option_kzt = 1; + kzt_effective_groups = 1; + check(0); + option_kzt_guest_tls = 1; + check(1); + option_kzt = 0; + check(0); + option_kzt = 2; + check(1); + kzt_effective_groups = 0; + check(0); + option_kzt_guest_tls = 0; + kzt_effective_groups = 1; + check(0); + puts("kzt Guest TLS opt-in policy: PASS"); + return 0; +} diff --git a/tests/unit/kzt/test_kzt_public_loader_observer.c b/tests/unit/kzt/test_kzt_public_loader_observer.c index 4ba3ee2694..bc65161c4e 100644 --- a/tests/unit/kzt/test_kzt_public_loader_observer.c +++ b/tests/unit/kzt/test_kzt_public_loader_observer.c @@ -28,7 +28,24 @@ #define SYMBOL_HASH_ADDR (SYMBOL_ELF_BASE + 0x200) #define SYMBOL_TABLE_ADDR (SYMBOL_ELF_BASE + 0x300) #define SYMBOL_STRING_ADDR (SYMBOL_ELF_BASE + 0x400) +#define TLS_ELF_BASE (FIXTURE_BASE + 0x9000) +#define TLS_DYNAMIC_ADDR (TLS_ELF_BASE + 0x300) +#define TLS_RELA_ADDR (TLS_ELF_BASE + 0x500) +#define TLS_IMAGE_ADDR (TLS_ELF_BASE + 0x600) +#define TLS_MODULE_RELOCATION_ADDR (TLS_ELF_BASE + 0x700) +#define TLS_EXTERNAL_RELOCATION_ADDR (TLS_ELF_BASE + 0x708) +#define TLS_STATIC_RELOCATION_ADDR (TLS_ELF_BASE + 0x710) +#define TLS_SYMBOL_TABLE_ADDR (TLS_ELF_BASE + 0x800) +#define TLS_STRING_TABLE_ADDR (TLS_ELF_BASE + 0x880) +#define TLS_HASH_ADDR (TLS_ELF_BASE + 0x8c0) +#define DUP_TLS_DYNAMIC_ADDR (FIXTURE_BASE + 0x5800) +#define DUP_TLS_HASH_ADDR (FIXTURE_BASE + 0x5900) +#define DUP_TLS_SYMBOL_ADDR (FIXTURE_BASE + 0x5a00) +#define DUP_TLS_STRING_ADDR (FIXTURE_BASE + 0x5b00) #define TEST_PAGE_SIZE 0x1000 +#define KZT_TEST_ET_DYN 3 +#define KZT_TEST_EM_X86_64 62 +#define KZT_TEST_PT_LOAD 1 #define KZT_TEST_PT_GNU_RELRO UINT32_C(0x6474e552) typedef struct test_x86_64_elf_header { @@ -68,6 +85,12 @@ typedef struct test_x86_64_symbol { uint64_t size; } test_x86_64_symbol_t; +typedef struct test_x86_64_relocation { + uint64_t offset; + uint64_t info; + int64_t addend; +} test_x86_64_relocation_t; + #define CHECK(condition) \ do { \ if (!(condition)) { \ @@ -231,14 +254,75 @@ static void write_elf_without_relro(fixture_t *fixture, fixture_write(fixture, load_bias + header.phoff, &phdr, sizeof(phdr)); } -static void write_gnu_hash_dlopen_object(fixture_t *fixture) +static void write_elf_loads( + fixture_t *fixture, + uintptr_t load_bias, + const test_x86_64_program_header_t *phdrs, + size_t phnum) +{ + test_x86_64_elf_header_t header = { 0 }; + + CHECK(phnum > 0); + CHECK(phnum <= UINT16_MAX); + header.ident[0] = 0x7f; + header.ident[1] = 'E'; + header.ident[2] = 'L'; + header.ident[3] = 'F'; + header.ident[4] = 2; + header.ident[5] = 1; + header.ident[6] = 1; + header.type = KZT_TEST_ET_DYN; + header.machine = KZT_TEST_EM_X86_64; + header.version = 1; + header.phoff = sizeof(header); + header.ehsize = sizeof(header); + header.phentsize = sizeof(*phdrs); + header.phnum = phnum; + fixture_write(fixture, load_bias, &header, sizeof(header)); + fixture_write(fixture, load_bias + header.phoff, + phdrs, phnum * sizeof(*phdrs)); +} + +static void write_symbol_object(fixture_t *fixture) +{ + const kzt_x86_64_dynamic_entry_t dynamic[] = { + { .tag = 4, .value = SYMBOL_HASH_ADDR - SYMBOL_ELF_BASE }, + { .tag = 5, .value = SYMBOL_STRING_ADDR - SYMBOL_ELF_BASE }, + { .tag = 6, .value = SYMBOL_TABLE_ADDR - SYMBOL_ELF_BASE }, + { .tag = 10, .value = sizeof("\0_dl_allocate_tls") }, + { .tag = 11, .value = sizeof(test_x86_64_symbol_t) }, + { .tag = KZT_X86_64_DT_NULL, .value = 0 }, + }; + const uint32_t hash_header[] = { 1, 2 }; + const test_x86_64_symbol_t symbols[] = { + { 0 }, + { + .name = 1, + .info = 0x12, + .section_index = 1, + .value = 0x1234, + }, + }; + static const char strings[] = "\0_dl_allocate_tls"; + + fixture_write(fixture, SYMBOL_DYNAMIC_ADDR, + dynamic, sizeof(dynamic)); + fixture_write(fixture, SYMBOL_HASH_ADDR, + hash_header, sizeof(hash_header)); + fixture_write(fixture, SYMBOL_TABLE_ADDR, + symbols, sizeof(symbols)); + fixture_write(fixture, SYMBOL_STRING_ADDR, + strings, sizeof(strings)); +} + +static void write_gnu_hash_symbol_object(fixture_t *fixture) { const kzt_x86_64_dynamic_entry_t dynamic[] = { { .tag = INT64_C(0x6ffffef5), .value = SYMBOL_HASH_ADDR - SYMBOL_ELF_BASE }, { .tag = 5, .value = SYMBOL_STRING_ADDR - SYMBOL_ELF_BASE }, { .tag = 6, .value = SYMBOL_TABLE_ADDR - SYMBOL_ELF_BASE }, - { .tag = 10, .value = sizeof("\0dlopen") }, + { .tag = 10, .value = sizeof("\0_dl_allocate_tls") }, { .tag = 11, .value = sizeof(test_x86_64_symbol_t) }, { .tag = KZT_X86_64_DT_NULL, .value = 0 }, }; @@ -263,10 +347,10 @@ static void write_gnu_hash_dlopen_object(fixture_t *fixture) .name = 1, .info = 0x12, .section_index = 1, - .value = 0x905d0, + .value = 0x1234, }, }; - static const char strings[] = "\0dlopen"; + static const char strings[] = "\0_dl_allocate_tls"; fixture_write(fixture, SYMBOL_DYNAMIC_ADDR, dynamic, sizeof(dynamic)); @@ -277,6 +361,145 @@ static void write_gnu_hash_dlopen_object(fixture_t *fixture) strings, sizeof(strings)); } +static void write_tls_object(fixture_t *fixture) +{ + test_x86_64_elf_header_t header = { 0 }; + const test_x86_64_program_header_t phdr = { + .type = 7, + .vaddr = TLS_IMAGE_ADDR - TLS_ELF_BASE + 8, + .filesz = 24, + .memsz = 32, + .align = 64, + }; + const kzt_x86_64_dynamic_entry_t dynamic[] = { + { .tag = 7, .value = TLS_RELA_ADDR - TLS_ELF_BASE }, + { .tag = 8, .value = 6 * sizeof(test_x86_64_relocation_t) }, + { .tag = 9, .value = sizeof(test_x86_64_relocation_t) }, + { .tag = 6, .value = TLS_SYMBOL_TABLE_ADDR - TLS_ELF_BASE }, + { .tag = 11, .value = sizeof(test_x86_64_symbol_t) }, + { .tag = 5, .value = TLS_STRING_TABLE_ADDR - TLS_ELF_BASE }, + { .tag = 10, .value = sizeof("\0tls_symbol") }, + { .tag = 4, .value = TLS_HASH_ADDR - TLS_ELF_BASE }, + { .tag = KZT_X86_64_DT_NULL, .value = 0 }, + }; + const test_x86_64_relocation_t relocations[] = { + { + .offset = TLS_EXTERNAL_RELOCATION_ADDR - TLS_ELF_BASE, + .info = (UINT64_C(1) << 32) | 16, + }, + { + .offset = TLS_MODULE_RELOCATION_ADDR - TLS_ELF_BASE, + .info = 16, + }, + { + .offset = TLS_STATIC_RELOCATION_ADDR - TLS_ELF_BASE, + .info = (UINT64_C(1) << 32) | 18, + .addend = 4, + }, + { + .offset = TLS_IMAGE_ADDR + 8 - TLS_ELF_BASE, + .info = 8, + .addend = 0x1234, + }, + { + .offset = TLS_IMAGE_ADDR + 16 - TLS_ELF_BASE, + .info = (UINT64_C(1) << 32) | 1, + }, + { + .offset = TLS_IMAGE_ADDR + 24 - TLS_ELF_BASE, + .info = (UINT64_C(2) << 32) | 1, + }, + }; + const test_x86_64_symbol_t symbols[] = { + { 0 }, + { + .name = 1, + .info = 0x16, + /* A default-visible definition can be preempted. */ + .section_index = 1, + }, + { + .info = 0x0a, + .section_index = 1, + .value = 0x555, + }, + }; + const uint64_t module_id = 7; + const uint64_t external_module_id = 3; + const int64_t static_relocation = -0x11c; + const uint64_t image_values[] = { + UINT64_C(0x123456789abcdef0), + UINT64_C(0xabcdef0123456789), + UINT64_C(0xfeedfacecafebeef), + }; + static const char string_table[] = "\0tls_symbol"; + const uint32_t hash_header[] = { 1, 3 }; + + header.ident[0] = 0x7f; + header.ident[1] = 'E'; + header.ident[2] = 'L'; + header.ident[3] = 'F'; + header.ident[4] = 2; + header.ident[5] = 1; + header.ident[6] = 1; + header.phoff = sizeof(header); + header.ehsize = sizeof(header); + header.phentsize = sizeof(phdr); + header.phnum = 1; + fixture_write(fixture, TLS_ELF_BASE, &header, sizeof(header)); + fixture_write(fixture, TLS_ELF_BASE + header.phoff, + &phdr, sizeof(phdr)); + fixture_write(fixture, TLS_DYNAMIC_ADDR, dynamic, sizeof(dynamic)); + fixture_write(fixture, TLS_RELA_ADDR, + relocations, sizeof(relocations)); + fixture_write(fixture, TLS_SYMBOL_TABLE_ADDR, + symbols, sizeof(symbols)); + fixture_write(fixture, TLS_STRING_TABLE_ADDR, + string_table, sizeof(string_table)); + fixture_write(fixture, TLS_HASH_ADDR, + hash_header, sizeof(hash_header)); + fixture_write(fixture, TLS_MODULE_RELOCATION_ADDR, + &module_id, sizeof(module_id)); + fixture_write(fixture, TLS_EXTERNAL_RELOCATION_ADDR, + &external_module_id, sizeof(external_module_id)); + fixture_write(fixture, TLS_STATIC_RELOCATION_ADDR, + &static_relocation, sizeof(static_relocation)); + fixture_write(fixture, TLS_IMAGE_ADDR + 8, + image_values, sizeof(image_values)); +} + +static void write_duplicate_tls_symbol(fixture_t *fixture) +{ + const kzt_x86_64_dynamic_entry_t dynamic[] = { + { .tag = 4, .value = DUP_TLS_HASH_ADDR }, + { .tag = 5, .value = DUP_TLS_STRING_ADDR }, + { .tag = 6, .value = DUP_TLS_SYMBOL_ADDR }, + { .tag = 10, .value = sizeof("\0tls_symbol") }, + { .tag = 11, .value = sizeof(test_x86_64_symbol_t) }, + { .tag = KZT_X86_64_DT_NULL, .value = 0 }, + }; + const uint32_t hash_header[] = { 1, 2 }; + const test_x86_64_symbol_t symbols[] = { + { 0 }, + { + .name = 1, + .info = 0x16, + .section_index = 1, + .value = 8, + }, + }; + static const char strings[] = "\0tls_symbol"; + + fixture_write(fixture, DUP_TLS_DYNAMIC_ADDR, + dynamic, sizeof(dynamic)); + fixture_write(fixture, DUP_TLS_HASH_ADDR, + hash_header, sizeof(hash_header)); + fixture_write(fixture, DUP_TLS_SYMBOL_ADDR, + symbols, sizeof(symbols)); + fixture_write(fixture, DUP_TLS_STRING_ADDR, + strings, sizeof(strings)); +} + static void setup_two_maps(fixture_t *fixture) { memset(fixture, 0, sizeof(*fixture)); @@ -363,12 +586,15 @@ static void test_deleted_address_can_be_observed_again(void) .read_memory = fixture_read, .opaque = &fixture, }; + uint64_t first_generation; setup_two_maps(&fixture); kzt_public_loader_observer_reset(&observer); CHECK(kzt_public_loader_observer_activate( &observer, DYNAMIC_ADDR, 16, &reader, record_visit, &log) == KZT_PUBLIC_LOADER_OK); + first_generation = observer.live_map_generations[1]; + CHECK(first_generation != 0); write_map(&fixture, MAP1_ADDR, UINT64_C(0x400000), NAME1_ADDR, FIXTURE_BASE + 0x5000, 0, 0); @@ -389,6 +615,7 @@ static void test_deleted_address_can_be_observed_again(void) CHECK(log.count == 1); CHECK(log.objects[0].link_map_addr == MAP2_ADDR); CHECK(log.objects[0].load_bias == UINT64_C(0xa00000)); + CHECK(observer.live_map_generations[1] != first_generation); } static void test_cycle_does_not_replace_last_complete_snapshot(void) @@ -606,6 +833,691 @@ static void test_object_relro_classification_uses_guest_memory(void) CHECK(has_relro == 0); } +static void test_symbol_lookup_uses_live_relocated_elf_state(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + uintptr_t symbol_addr = 0; + + memset(&fixture, 0, sizeof(fixture)); + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, SYMBOL_ELF_BASE, NAME1_ADDR, + SYMBOL_DYNAMIC_ADDR, 0, 0); + write_symbol_object(&fixture); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + + CHECK(kzt_public_loader_find_symbol( + &observer, &reader, "_dl_allocate_tls", &symbol_addr) == + KZT_PUBLIC_LOADER_OK); + CHECK(symbol_addr == SYMBOL_ELF_BASE + 0x1234); + CHECK(log.count == 1); + symbol_addr = 0; + CHECK(kzt_public_loader_find_symbol_in_object( + &log.objects[0], &reader, + "_dl_allocate_tls", &symbol_addr) == + KZT_PUBLIC_LOADER_OK); + CHECK(symbol_addr == SYMBOL_ELF_BASE + 0x1234); + CHECK(kzt_public_loader_find_symbol( + &observer, &reader, "missing", &symbol_addr) == + KZT_PUBLIC_LOADER_NOT_FOUND); + + memset(&fixture, 0, sizeof(fixture)); + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, SYMBOL_ELF_BASE, NAME1_ADDR, + SYMBOL_DYNAMIC_ADDR, 0, 0); + write_gnu_hash_symbol_object(&fixture); + kzt_public_loader_observer_reset(&observer); + log.count = 0; + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + CHECK(kzt_public_loader_find_symbol( + &observer, &reader, "_dl_allocate_tls", &symbol_addr) == + KZT_PUBLIC_LOADER_OK); + CHECK(symbol_addr == SYMBOL_ELF_BASE + 0x1234); +} + +static void test_address_lookup_uses_exact_load_segments(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + kzt_public_loader_object_t object = { 0 }; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + const test_x86_64_program_header_t elf1_phdrs[] = { + { + .type = KZT_TEST_PT_LOAD, + .offset = 0x1000, + .vaddr = 0x1000, + .filesz = 0x400, + .memsz = 0x800, + .align = 0x1000, + }, + { + .type = KZT_TEST_PT_LOAD, + .offset = 0x3000, + .vaddr = 0x3000, + .filesz = 0x500, + .memsz = 0x500, + .align = 0x1000, + }, + { + .type = KZT_TEST_PT_LOAD, + .offset = 0x3200, + .vaddr = 0x3200, + .filesz = 0x200, + .memsz = 0x200, + .align = 0x100, + }, + }; + const test_x86_64_program_header_t elf2_phdr = { + .type = KZT_TEST_PT_LOAD, + .offset = 0x5000, + .vaddr = 0x5000, + .filesz = 0x100, + .memsz = 0x100, + .align = 0x1000, + }; + + memset(&fixture, 0, sizeof(fixture)); + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, ELF1_BASE, NAME1_ADDR, + FIXTURE_BASE + 0x5000, MAP2_ADDR, 0); + write_map(&fixture, MAP2_ADDR, ELF2_BASE, NAME2_ADDR, + FIXTURE_BASE + 0x5100, 0, MAP1_ADDR); + write_elf_loads(&fixture, ELF1_BASE, elf1_phdrs, + sizeof(elf1_phdrs) / sizeof(elf1_phdrs[0])); + write_elf_loads(&fixture, ELF2_BASE, &elf2_phdr, 1); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x1100, &object) == + KZT_PUBLIC_LOADER_OK); + CHECK(object.link_map_addr == MAP1_ADDR); + CHECK(object.load_bias == ELF1_BASE); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x1600, &object) == + KZT_PUBLIC_LOADER_OK); + CHECK(object.link_map_addr == MAP1_ADDR); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x3300, &object) == + KZT_PUBLIC_LOADER_OK); + CHECK(object.link_map_addr == MAP1_ADDR); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x2000, &object) == + KZT_PUBLIC_LOADER_NOT_FOUND); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x1800, &object) == + KZT_PUBLIC_LOADER_NOT_FOUND); +} + +static void test_address_lookup_rejects_ambiguity_and_malformed_elf(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + kzt_public_loader_object_t object = { 0 }; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + test_x86_64_program_header_t phdr1 = { + .type = KZT_TEST_PT_LOAD, + .offset = 0x7000, + .vaddr = 0x7000, + .filesz = 0x200, + .memsz = 0x200, + .align = 0x1000, + }; + const test_x86_64_program_header_t phdr2 = { + .type = KZT_TEST_PT_LOAD, + .offset = 0x2000, + .vaddr = 0x2000, + .filesz = 0x200, + .memsz = 0x200, + .align = 0x1000, + }; + test_x86_64_elf_header_t invalid_header = { 0 }; + + memset(&fixture, 0, sizeof(fixture)); + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, ELF1_BASE, NAME1_ADDR, + FIXTURE_BASE + 0x5000, MAP2_ADDR, 0); + write_map(&fixture, MAP2_ADDR, ELF2_BASE, NAME2_ADDR, + FIXTURE_BASE + 0x5100, 0, MAP1_ADDR); + write_elf_loads(&fixture, ELF1_BASE, &phdr1, 1); + write_elf_loads(&fixture, ELF2_BASE, &phdr2, 1); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x7100, &object) == + KZT_PUBLIC_LOADER_INVALID_STATE); + + write_map(&fixture, MAP1_ADDR, ELF1_BASE, NAME1_ADDR, + FIXTURE_BASE + 0x5000, 0, 0); + CHECK(kzt_public_loader_observer_refresh( + &observer, &reader, record_visit, &log) == + KZT_PUBLIC_LOADER_OK); + fixture_write(&fixture, ELF1_BASE, &invalid_header, + sizeof(invalid_header)); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x7100, &object) == + KZT_PUBLIC_LOADER_INVALID_STATE); + + write_elf_loads(&fixture, ELF1_BASE, &phdr1, 1); + phdr1.vaddr = UINT64_MAX; + phdr1.align = 1; + fixture_write(&fixture, ELF1_BASE + sizeof(invalid_header), + &phdr1, sizeof(phdr1)); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x7100, &object) == + KZT_PUBLIC_LOADER_OVERFLOW); +} + +static void test_address_lookup_requires_active_consistent_observer(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + kzt_public_loader_object_t object = { 0 }; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + + memset(&fixture, 0, sizeof(fixture)); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x1000, &object) == + KZT_PUBLIC_LOADER_INVALID_INPUT); + + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, 0, NAME1_ADDR, + FIXTURE_BASE + 0x5000, 0, 0); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x1000, &object) == + KZT_PUBLIC_LOADER_NOT_FOUND); + + write_debug(&fixture, KZT_LOADER_DEBUG_ADD, MAP1_ADDR); + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF1_BASE + 0x1000, &object) == + KZT_PUBLIC_LOADER_BUSY); +} + +static void test_loader_state_probe_requires_same_consistent_instance(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + + setup_two_maps(&fixture); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + CHECK(kzt_public_loader_state_is_consistent( + &observer, &reader) == KZT_PUBLIC_LOADER_OK); + + write_debug(&fixture, KZT_LOADER_DEBUG_ADD, MAP1_ADDR); + CHECK(kzt_public_loader_state_is_consistent( + &observer, &reader) == KZT_PUBLIC_LOADER_BUSY); + + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + observer.r_brk_addr += 8; + CHECK(kzt_public_loader_state_is_consistent( + &observer, &reader) == KZT_PUBLIC_LOADER_INVALID_STATE); +} + +static void test_address_lookup_accepts_remembered_object_during_add(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + kzt_public_loader_object_t object = { 0 }; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + const test_x86_64_program_header_t phdr = { + .type = KZT_TEST_PT_LOAD, + .offset = 0x1000, + .vaddr = 0x1000, + .filesz = 0x400, + .memsz = 0x800, + .align = 0x1000, + }; + + memset(&fixture, 0, sizeof(fixture)); + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, ELF1_BASE, NAME1_ADDR, + FIXTURE_BASE + 0x5000, 0, 0); + write_elf_loads(&fixture, ELF1_BASE, &phdr, 1); + write_elf_loads(&fixture, ELF2_BASE, &phdr, 1); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + + /* + * The pre-RELRO path can observe and remember a new object while ld.so + * still reports RT_ADD. Address ownership for that already-mapped object + * must not wait for the later RT_CONSISTENT notification. + */ + write_map(&fixture, MAP1_ADDR, ELF1_BASE, NAME1_ADDR, + FIXTURE_BASE + 0x5000, MAP2_ADDR, 0); + write_map(&fixture, MAP2_ADDR, ELF2_BASE, NAME2_ADDR, + FIXTURE_BASE + 0x5100, 0, MAP1_ADDR); + CHECK(kzt_public_loader_observer_remember(&observer, MAP2_ADDR) == + KZT_PUBLIC_LOADER_OK); + write_debug(&fixture, KZT_LOADER_DEBUG_ADD, MAP1_ADDR); + log.count = 0; + CHECK(kzt_public_loader_observer_refresh( + &observer, &reader, record_visit, &log) == + KZT_PUBLIC_LOADER_BUSY); + CHECK(log.count == 0); + + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, ELF2_BASE + 0x1100, &object) == + KZT_PUBLIC_LOADER_OK); + CHECK(object.link_map_addr == MAP2_ADDR); + CHECK(object.load_bias == ELF2_BASE); +} + +static void test_address_lookup_walks_beyond_snapshot_capacity(void) +{ + enum { + object_count = KZT_PUBLIC_LOADER_MAX_OBJECTS + 1, + elf_stride = 0x80, + }; + const uintptr_t map_base = FIXTURE_BASE + 0x3000; + const uintptr_t elf_base = FIXTURE_BASE + 0x6000; + const test_x86_64_program_header_t phdr = { + .type = KZT_TEST_PT_LOAD, + .offset = 0, + .vaddr = 0, + .filesz = elf_stride, + .memsz = elf_stride, + .align = 1, + }; + fixture_t fixture; + kzt_public_loader_observer_t observer; + kzt_public_loader_object_t object = { 0 }; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + + memset(&fixture, 0, sizeof(fixture)); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, map_base); + kzt_public_loader_observer_reset(&observer); + observer.active = 1; + observer.r_debug_addr = R_DEBUG_ADDR; + observer.r_brk_addr = R_BRK_ADDR; + observer.live_map_count = KZT_PUBLIC_LOADER_MAX_OBJECTS; + + for (size_t index = 0; index < object_count; ++index) { + uintptr_t map_addr = + map_base + index * sizeof(kzt_x86_64_link_map_prefix_t); + uintptr_t object_base = elf_base + index * elf_stride; + uintptr_t next = index + 1 < object_count + ? map_addr + sizeof(kzt_x86_64_link_map_prefix_t) : 0; + uintptr_t previous = index + ? map_addr - sizeof(kzt_x86_64_link_map_prefix_t) : 0; + + write_map(&fixture, map_addr, object_base, 0, 0, + next, previous); + write_elf_loads(&fixture, object_base, &phdr, 1); + if (index < KZT_PUBLIC_LOADER_MAX_OBJECTS) { + observer.live_maps[index] = map_addr; + } + } + + CHECK(kzt_public_loader_find_object_by_address( + &observer, &reader, + elf_base + (object_count - 1) * elf_stride + 0x70, + &object) == KZT_PUBLIC_LOADER_OK); + CHECK(object.link_map_addr == + map_base + (object_count - 1) * + sizeof(kzt_x86_64_link_map_prefix_t)); + CHECK(object.load_bias == + elf_base + (object_count - 1) * elf_stride); +} + +static void test_tls_collection_uses_live_image_and_module_relocation(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + kzt_public_loader_tls_object_t tls_objects[2]; + size_t tls_count = 0; + uint64_t materialized_image[3]; + + memset(&fixture, 0, sizeof(fixture)); + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, TLS_ELF_BASE, NAME1_ADDR, + TLS_DYNAMIC_ADDR, 0, 0); + write_tls_object(&fixture); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + + CHECK(kzt_public_loader_collect_tls( + &observer, &reader, tls_objects, 2, &tls_count) == + KZT_PUBLIC_LOADER_OK); + CHECK(tls_count == 1); + CHECK(tls_objects[0].image_addr == TLS_IMAGE_ADDR + 8); + CHECK(tls_objects[0].file_size == 24); + CHECK(tls_objects[0].memory_size == 32); + CHECK(tls_objects[0].alignment == 64); + CHECK(tls_objects[0].first_byte_offset == 8); + CHECK(tls_objects[0].static_tls_offset == -0x120); + CHECK(tls_objects[0].static_tls_offset_valid == 1); + CHECK(tls_objects[0].static_tls_offset_needs_validation == 0); + CHECK(tls_objects[0].static_tls_symbol_name_addr == 0); + CHECK(tls_objects[0].module_id == 7); + CHECK(tls_objects[0].link_map_addr == MAP1_ADDR); + CHECK(tls_objects[0].load_generation != 0); + CHECK(kzt_public_loader_materialize_tls_image( + &tls_objects[0], &reader, &materialized_image, + sizeof(materialized_image)) == KZT_PUBLIC_LOADER_OK); + CHECK(materialized_image[0] == TLS_ELF_BASE + 0x1234); + CHECK(materialized_image[1] == UINT64_C(0xabcdef0123456789)); + CHECK(materialized_image[2] == UINT64_C(0xfeedfacecafebeef)); + CHECK(kzt_public_loader_materialize_tls_image( + &tls_objects[0], &reader, materialized_image, + sizeof(materialized_image) - 1) == + KZT_PUBLIC_LOADER_INVALID_INPUT); + fixture.reject_reads = 1; + CHECK(kzt_public_loader_materialize_tls_image( + &tls_objects[0], &reader, materialized_image, + sizeof(materialized_image)) == + KZT_PUBLIC_LOADER_READ_ERROR); + fixture.reject_reads = 0; + + { + test_x86_64_relocation_t original; + test_x86_64_relocation_t unsupported; + uintptr_t relocation_addr = + TLS_RELA_ADDR + 5 * sizeof(test_x86_64_relocation_t); + + CHECK(fixture_read(relocation_addr, &original, + sizeof(original), &fixture) == 0); + unsupported = original; + unsupported.info = 99; + fixture_write(&fixture, relocation_addr, + &unsupported, sizeof(unsupported)); + CHECK(kzt_public_loader_materialize_tls_image( + &tls_objects[0], &reader, materialized_image, + sizeof(materialized_image)) == + KZT_PUBLIC_LOADER_INVALID_STATE); + fixture_write(&fixture, relocation_addr, + &original, sizeof(original)); + } + + { + const uint64_t unresolved_module_id = 0; + const uint64_t resolved_module_id = 7; + + fixture_write(&fixture, TLS_MODULE_RELOCATION_ADDR, + &unresolved_module_id, + sizeof(unresolved_module_id)); + tls_count = 0; + CHECK(kzt_public_loader_collect_tls( + &observer, &reader, tls_objects, 2, &tls_count) == + KZT_PUBLIC_LOADER_OK); + CHECK(tls_count == 1); + CHECK(tls_objects[0].module_id == 0); + fixture_write(&fixture, TLS_MODULE_RELOCATION_ADDR, + &resolved_module_id, + sizeof(resolved_module_id)); + } + { + const int64_t pending_static_offset = 0; + const int64_t resolved_static_offset = -0x11c; + + fixture_write(&fixture, TLS_STATIC_RELOCATION_ADDR, + &pending_static_offset, + sizeof(pending_static_offset)); + CHECK(kzt_public_loader_collect_tls( + &observer, &reader, tls_objects, 2, &tls_count) == + KZT_PUBLIC_LOADER_BUSY); + fixture_write(&fixture, TLS_STATIC_RELOCATION_ADDR, + &resolved_static_offset, + sizeof(resolved_static_offset)); + } + + write_debug(&fixture, KZT_LOADER_DEBUG_ADD, MAP1_ADDR); + tls_count = 0; + CHECK(kzt_public_loader_snapshot_tls( + &observer, 0, 0, &reader, 0, + tls_objects, 2, &tls_count) == KZT_PUBLIC_LOADER_OK); + CHECK(tls_count == 1); + CHECK(tls_objects[0].module_id == 7); + CHECK(tls_objects[0].static_tls_offset == -0x120); +} + +static void test_tls_snapshot_walks_beyond_observer_capacity(void) +{ + enum { object_count = KZT_PUBLIC_LOADER_MAX_OBJECTS + 1 }; + const uintptr_t map_base = FIXTURE_BASE + 0x3000; + const uintptr_t tls_map_addr = + map_base + (object_count - 1) * + sizeof(kzt_x86_64_link_map_prefix_t); + fixture_t fixture; + kzt_public_loader_observer_t observer; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + kzt_public_loader_tls_object_t tls_objects[2]; + size_t tls_count = 0; + uint64_t first_generation; + + memset(&fixture, 0, sizeof(fixture)); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, map_base); + kzt_public_loader_observer_reset(&observer); + observer.active = 1; + observer.r_debug_addr = R_DEBUG_ADDR; + observer.r_brk_addr = R_BRK_ADDR; + observer.live_map_count = KZT_PUBLIC_LOADER_MAX_OBJECTS; + observer.next_load_generation = KZT_PUBLIC_LOADER_MAX_OBJECTS; + + for (size_t index = 0; index < object_count; ++index) { + uintptr_t map_addr = + map_base + index * sizeof(kzt_x86_64_link_map_prefix_t); + uintptr_t next = index + 1 < object_count + ? map_addr + sizeof(kzt_x86_64_link_map_prefix_t) : 0; + uintptr_t previous = index + ? map_addr - sizeof(kzt_x86_64_link_map_prefix_t) : 0; + + write_map(&fixture, map_addr, + index + 1 == object_count ? TLS_ELF_BASE : 0, + 0, + index + 1 == object_count ? TLS_DYNAMIC_ADDR : 0, + next, previous); + if (index < KZT_PUBLIC_LOADER_MAX_OBJECTS) { + observer.live_maps[index] = map_addr; + observer.live_map_generations[index] = index + 1; + } + } + write_tls_object(&fixture); + + CHECK(kzt_public_loader_snapshot_tls( + &observer, 0, 0, &reader, 0, + tls_objects, 2, &tls_count) == KZT_PUBLIC_LOADER_OK); + CHECK(tls_count == 1); + CHECK(tls_objects[0].link_map_addr == tls_map_addr); + CHECK(tls_objects[0].module_id == 7); + CHECK(tls_objects[0].load_generation != 0); + first_generation = tls_objects[0].load_generation; + + tls_count = 0; + CHECK(kzt_public_loader_snapshot_tls( + &observer, 0, 0, &reader, 0, + tls_objects, 2, &tls_count) == KZT_PUBLIC_LOADER_OK); + CHECK(tls_count == 1); + CHECK(tls_objects[0].link_map_addr == tls_map_addr); + CHECK(tls_objects[0].load_generation == first_generation); +} + +static void test_tls_snapshot_does_not_consume_binding_visit(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + kzt_public_loader_tls_object_t tls_objects[2]; + size_t tls_count = 0; + + memset(&fixture, 0, sizeof(fixture)); + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, SYMBOL_ELF_BASE, NAME1_ADDR, + SYMBOL_DYNAMIC_ADDR, 0, 0); + write_symbol_object(&fixture); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + CHECK(observer.live_map_count == 1); + + write_map(&fixture, MAP1_ADDR, 0, NAME1_ADDR, + 0, MAP2_ADDR, 0); + write_map(&fixture, MAP2_ADDR, TLS_ELF_BASE, NAME2_ADDR, + TLS_DYNAMIC_ADDR, 0, MAP1_ADDR); + write_tls_object(&fixture); + CHECK(kzt_public_loader_snapshot_tls( + &observer, DYNAMIC_ADDR, 16, &reader, 0, + tls_objects, 2, &tls_count) == KZT_PUBLIC_LOADER_OK); + CHECK(tls_count == 1); + CHECK(tls_objects[0].module_id == 7); + CHECK(tls_objects[0].link_map_addr == MAP2_ADDR); + CHECK(tls_objects[0].load_generation != 0); + CHECK(observer.live_map_count == 1); + + log.count = 0; + CHECK(kzt_public_loader_observer_refresh( + &observer, &reader, record_visit, &log) == + KZT_PUBLIC_LOADER_OK); + CHECK(log.count == 1); + CHECK(log.objects[0].link_map_addr == MAP2_ADDR); +} + +static void test_default_visible_tls_requires_unique_owner(void) +{ + fixture_t fixture; + visit_log_t log = { 0 }; + kzt_public_loader_observer_t observer; + const kzt_public_loader_reader_t reader = { + .read_memory = fixture_read, + .opaque = &fixture, + }; + kzt_public_loader_tls_object_t tls_objects[2]; + size_t tls_count = 0; + + memset(&fixture, 0, sizeof(fixture)); + write_dynamic(&fixture, R_DEBUG_ADDR); + write_debug(&fixture, KZT_LOADER_DEBUG_CONSISTENT, MAP1_ADDR); + write_map(&fixture, MAP1_ADDR, TLS_ELF_BASE, NAME1_ADDR, + TLS_DYNAMIC_ADDR, MAP2_ADDR, 0); + write_map(&fixture, MAP2_ADDR, 0, NAME2_ADDR, + DUP_TLS_DYNAMIC_ADDR, 0, MAP1_ADDR); + write_tls_object(&fixture); + write_duplicate_tls_symbol(&fixture); + kzt_public_loader_observer_reset(&observer); + CHECK(kzt_public_loader_observer_activate( + &observer, DYNAMIC_ADDR, 16, &reader, + record_visit, &log) == KZT_PUBLIC_LOADER_OK); + + CHECK(kzt_public_loader_collect_tls( + &observer, &reader, tls_objects, 2, &tls_count) == + KZT_PUBLIC_LOADER_INVALID_STATE); +} + +static void write_gnu_hash_dlopen_object(fixture_t *fixture) +{ + const kzt_x86_64_dynamic_entry_t dynamic[] = { + { .tag = INT64_C(0x6ffffef5), + .value = SYMBOL_HASH_ADDR - SYMBOL_ELF_BASE }, + { .tag = 5, .value = SYMBOL_STRING_ADDR - SYMBOL_ELF_BASE }, + { .tag = 6, .value = SYMBOL_TABLE_ADDR - SYMBOL_ELF_BASE }, + { .tag = 10, .value = sizeof("\0dlopen") }, + { .tag = 11, .value = sizeof(test_x86_64_symbol_t) }, + { .tag = KZT_X86_64_DT_NULL, .value = 0 }, + }; + const struct { + uint32_t bucket_count; + uint32_t symbol_offset; + uint32_t bloom_size; + uint32_t bloom_shift; + uint64_t bloom; + uint32_t bucket; + uint32_t chain; + } hash = { + .bucket_count = 1, + .symbol_offset = 1, + .bloom_size = 1, + .bucket = 1, + .chain = 1, + }; + const test_x86_64_symbol_t symbols[] = { + { 0 }, + { + .name = 1, + .info = 0x12, + .section_index = 1, + .value = 0x905d0, + }, + }; + static const char strings[] = "\0dlopen"; + + fixture_write(fixture, SYMBOL_DYNAMIC_ADDR, + dynamic, sizeof(dynamic)); + fixture_write(fixture, SYMBOL_HASH_ADDR, &hash, sizeof(hash)); + fixture_write(fixture, SYMBOL_TABLE_ADDR, + symbols, sizeof(symbols)); + fixture_write(fixture, SYMBOL_STRING_ADDR, + strings, sizeof(strings)); +} + static void test_symbol_lookup_uses_live_gnu_hash_object(void) { fixture_t fixture; @@ -649,6 +1561,17 @@ int main(void) test_observed_processed_and_reported_states_are_distinct(); test_object_relro_classification_uses_guest_memory(); test_symbol_lookup_uses_live_gnu_hash_object(); + test_symbol_lookup_uses_live_relocated_elf_state(); + test_address_lookup_uses_exact_load_segments(); + test_address_lookup_rejects_ambiguity_and_malformed_elf(); + test_address_lookup_requires_active_consistent_observer(); + test_loader_state_probe_requires_same_consistent_instance(); + test_address_lookup_accepts_remembered_object_during_add(); + test_address_lookup_walks_beyond_snapshot_capacity(); + test_tls_collection_uses_live_image_and_module_relocation(); + test_tls_snapshot_walks_beyond_observer_capacity(); + test_tls_snapshot_does_not_consume_binding_visit(); + test_default_visible_tls_requires_unique_owner(); puts("kzt public loader observer tests: PASS"); return 0; } diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 3c2bc13451..36a460c6c6 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -202,6 +202,41 @@ test( suite: 'lat-pr-fast', ) +test_kzt_guest_tls_policy = executable( + 'test-kzt-guest-tls-policy', + files('kzt/test_kzt_guest_tls_policy.c'), + include_directories: include_directories('../../target/i386/latx/include'), + build_by_default: false, +) +test('test-kzt-guest-tls-policy', test_kzt_guest_tls_policy, suite: 'lat-pr-fast') + +test_kzt_guest_tls_epoch = executable( + 'test-kzt-guest-tls-epoch', + files('kzt/test_kzt_guest_tls_epoch.c'), + include_directories: include_directories( + '../../target/i386/latx/include', + ), + build_by_default: false, +) + +test( + 'test-kzt-guest-tls-epoch', + test_kzt_guest_tls_epoch, + suite: 'lat-pr-fast', +) + +test( + 'check-kzt-tls-refresh-boundary', + python, + args: [ + files('kzt/check_kzt_tls_refresh_boundary.py'), + project_source_root / 'target/i386/latx/context/callback.c', + project_source_root / 'target/i386/latx/context/wrappedlibdl.c', + project_source_root / 'target/i386/latx/context/myalign.c', + ], + suite: 'lat-pr-fast', +) + test_kzt_relocation_transaction = executable( 'test-kzt-relocation-transaction', files( From e97cb175aafec1f9018625bbd8569e729b33cdfc Mon Sep 17 00:00:00 2001 From: Hanlu Li Date: Sun, 6 Sep 2026 17:00:28 +0800 Subject: [PATCH 5/5] LATX, feat: Add explicit libc state propagation at native-call boundaries Provide an initially inactive libc boundary broker for errno, h_errno and locale names. Keep Guest thread destruction in the TLS runtime. Consumers explicitly initialize and activate the broker; enabling Guest TLS alone does not enable semantic propagation. Keep private libc pointers within their owning domain. Own locale projections per Guest context, document borrowed handles, and release each projection in its owning libc during context teardown. The two-thread regression fails with the old shared cache and passes with separate contexts, including mutation and release of an owned duplicate. Signed-off-by: Hanlu Li --- docs/devel/kzt-libc-boundaries.md | 48 + linux-user/main.c | 2 + linux-user/syscall.c | 14 +- target/i386/cpu.h | 1 + target/i386/latx/context/callback.c | 32 +- target/i386/latx/context/kzt-libc-semantic.c | 1209 +++++++++++++++++ target/i386/latx/context/meson.build | 1 + target/i386/latx/context/myalign.c | 2 + target/i386/latx/context/wrappedlibc.c | 7 + target/i386/latx/include/callback.h | 5 + target/i386/latx/include/kzt-libc-semantic.h | 119 ++ tests/integration/kzt-libc-boundary.c | 200 +++ .../registrations/x11-kzt/meson.build | 10 + tests/integration/test-kzt-libc-boundary.sh | 32 + 14 files changed, 1678 insertions(+), 4 deletions(-) create mode 100644 docs/devel/kzt-libc-boundaries.md create mode 100644 target/i386/latx/context/kzt-libc-semantic.c create mode 100644 target/i386/latx/include/kzt-libc-semantic.h create mode 100644 tests/integration/kzt-libc-boundary.c create mode 100755 tests/integration/test-kzt-libc-boundary.sh diff --git a/docs/devel/kzt-libc-boundaries.md b/docs/devel/kzt-libc-boundaries.md new file mode 100644 index 0000000000..b50ea0bc8d --- /dev/null +++ b/docs/devel/kzt-libc-boundaries.md @@ -0,0 +1,48 @@ +# KZT libc state at explicit native-call boundaries + +Guest TLS storage and cross-libc state propagation are separate facilities. +This layer transfers errno/h_errno values and locale category names across +an explicitly selected native-call boundary. It does not transfer locale_t +objects, ctype pointers or allocator ownership between libc instances. + +Each Guest context owns its own non-global locale projection objects; they +are not process-wide cache handles. A current locale obtained with +`uselocale(0)` is borrowed from the boundary runtime and remains valid only +until that context is destroyed. Consumers must call `duplocale()` to obtain +an owned copy before modifying it with `newlocale(..., base)` or releasing +it with `freelocale()`. Borrowed projection handles must not be shared with +other threads. Context teardown deselects its active projections before +freeing them in their owning libc. Ordinary consumer-created locale objects +retain their original ownership. + +The caller provides the entry state. On return, the callee provides the +state to propagate back. Guest helper calls used by initialization must +not themselves initiate another semantic boundary. + +The broker starts inactive. A consumer first initializes semantic state +for the current Guest context and explicitly prepares the process locale +before invoking native code that requires this protocol. Guest TLS must +already be enabled. Merely enabling KZT or optional Guest TLS does not +activate libc state propagation. + +Native consumers explicitly call `kzt_libc_semantic_enter_current()` +and `kzt_libc_semantic_leave_current()` around a Guest-to-Host call. +Host-to-Guest calls that participate use +`latx_run_guest_callback_with_libc()`. The ordinary +`latx_run_guest_callback()`, `RunFunctionWithState()` and formatted +callback entries continue to provide TLS without projecting libc state, +even when another consumer has activated the process broker. + +Ordinary Guest pthread key/value operations and TLS destructor ownership +belong to the Guest thread runtime, not this broker. + +Locale names must be available in both libc installations. Failure to +reconstruct a required locale is an error, not permission to substitute +a pointer or silently select another locale. Resolver state, cancellation +and other private libc caches are not covered by this protocol. + +The extended resolver-isolation probe currently fails on the ABI1 profile: +attached threads can receive the same Guest `__res_state()` object. +Neither TLS opt-in nor this broker makes resolver APIs safe on attached +threads. This is a known limitation, retained in local extended validation, +not a successful test or a bidirectional resolver implementation. diff --git a/linux-user/main.c b/linux-user/main.c index 4c9080ff96..1fb58f415a 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -80,6 +80,7 @@ int mydebug = 1; #if defined(CONFIG_LATX_KZT) #include "kzt-groups.h" #include "kzt-guest-tls.h" +#include "kzt-libc-semantic.h" #include "wrappertbbridge.h" box64context_t* my_context = NULL; elfheader_t* elf_header = NULL; @@ -333,6 +334,7 @@ static CPUArchState *cpu_copy_into(CPUArchState *env, CPUState *new_cpu) new_env->kzt_guest_tls_allocation = NULL; new_env->kzt_guest_tls_parent_snapshot = NULL; new_env->kzt_guest_thread_state = NULL; + new_env->kzt_libc_semantic_state = NULL; #endif /* diff --git a/linux-user/syscall.c b/linux-user/syscall.c index 827771718b..63034d66f7 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -161,6 +161,7 @@ #include "lsenv.h" #include "kzt-guest-tls.h" #include "kzt-guest-thread.h" +#include "kzt-libc-semantic.h" #include "myalign.h" #include #include "aot.h" @@ -9647,6 +9648,7 @@ static void latx_host_thread_destructor(void *opaque) CPUState *cpu = opaque; kzt_guest_thread_destroy(cpu->env_ptr); + kzt_libc_semantic_destroy(cpu->env_ptr); if (kzt_guest_tls_cleanup_robust_list(cpu->env_ptr) != 0) { fprintf(stderr, "KZT Guest robust-list cleanup failed; " @@ -9724,6 +9726,7 @@ int latx_finalize_host_thread_template(CPUArchState *env) template_env->kzt_guest_tls_parent_snapshot = NULL; template_env->kzt_guest_tls_allocation = NULL; template_env->kzt_guest_thread_state = NULL; + template_env->kzt_libc_semantic_state = NULL; if (kzt_guest_tls_snapshot_parent(env, template_env) != 0) { latx_host_thread_release_cpu(template_cpu); goto out; @@ -9797,6 +9800,7 @@ static int latx_attach_current_host_thread_once(void) new_env->kzt_guest_tls_parent_snapshot = NULL; new_env->kzt_guest_tls_allocation = NULL; new_env->kzt_guest_thread_state = NULL; + new_env->kzt_libc_semantic_state = NULL; if (kzt_guest_tls_clone_parent_snapshot(parent_env, new_env) != 0) { latx_host_thread_release_cpu(new_cpu); rcu_unregister_thread(); @@ -9843,7 +9847,13 @@ static int latx_attach_current_host_thread_once(void) return ret; } kzt_guest_thread_initialize(new_env); - + if (kzt_libc_semantic_process_ready() && + kzt_libc_semantic_initialize(new_env) != 0) { + pthread_mutex_unlock(&clone_lock); + latx_host_thread_destructor(new_cpu); + pthread_setcancelstate(old_cancel_state, NULL); + return -1; + } if (pthread_setspecific(latx_host_thread_key, new_cpu) != 0) { pthread_mutex_unlock(&clone_lock); latx_host_thread_destructor(new_cpu); @@ -9933,6 +9943,7 @@ static void *clone_func(void *arg) static void cleanup_guest_thread_resources(CPUArchState *env) { + kzt_libc_semantic_destroy(env); assert(env->gdt.base); target_munmap(env->gdt.base, sizeof(uint64_t) * TARGET_GDT_ENTRIES, 0); } @@ -10214,6 +10225,7 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp, kzt_guest_tls_after_fork_child(env); kzt_guest_loader_after_fork_child(); kzt_guest_thread_after_fork_child(); + kzt_libc_semantic_after_fork_child(); #if defined(TARGET_NR_timer_create) posix_timer_fork_end(true); #endif diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 8b13f907be..fd4576f879 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -1675,6 +1675,7 @@ typedef struct CPUX86State { void *kzt_guest_tls_allocation; void *kzt_guest_thread_state; void *kzt_guest_tls_parent_snapshot; + void *kzt_libc_semantic_state; #endif #endif } CPUX86State; diff --git a/target/i386/latx/context/callback.c b/target/i386/latx/context/callback.c index c81b01987d..0a92878e41 100644 --- a/target/i386/latx/context/callback.c +++ b/target/i386/latx/context/callback.c @@ -20,6 +20,7 @@ #include "lsenv.h" #include "qemu.h" #include "kzt-guest-tls.h" +#include "kzt-libc-semantic.h" #ifdef TARGET_X86_64 typedef struct CallbackFrame { @@ -154,13 +155,15 @@ typedef enum LatxGuestCallKind { LATX_GUEST_USER_CALLBACK, LATX_GUEST_INTERNAL_HELPER, LATX_GUEST_INTERNAL_NO_REFRESH, + LATX_GUEST_LIBC_CALLBACK, } LatxGuestCallKind; #define LATX_GUEST_TLS_REFRESH_RETRIES 1000 static bool callback_is_user(LatxGuestCallKind kind) { - return kind == LATX_GUEST_USER_CALLBACK; + return kind == LATX_GUEST_USER_CALLBACK || + kind == LATX_GUEST_LIBC_CALLBACK; } static int callback_disable_cancellation(LatxGuestCallKind kind, @@ -198,6 +201,7 @@ typedef struct CallbackScope { int cancellation_disabled; bool execution_entered; bool internal_entered; + bool semantic_entered; } CallbackScope; static void callback_scope_leave(CallbackScope *scope) @@ -212,7 +216,9 @@ static void callback_scope_leave(CallbackScope *scope) } /* Restoring cancellation may not return; cleanup must be idempotent. */ *scope = (CallbackScope) { 0 }; - + if (current.semantic_entered) { + kzt_libc_semantic_host_to_guest_leave(current.cpu); + } if (current.execution_entered) { kzt_guest_tls_execution_leave(current.cpu); } @@ -270,7 +276,15 @@ static int callback_scope_enter(CallbackScope *scope, LatxGuestCallKind kind) if (callback_is_user(kind)) { kzt_guest_tls_execution_enter(scope->cpu); scope->execution_entered = true; - + if (kind == LATX_GUEST_LIBC_CALLBACK) { + if (!kzt_libc_semantic_process_ready()) { + result = -1; + goto out; + } + result = kzt_libc_semantic_host_to_guest_enter( + scope->cpu, entry_errno, entry_h_errno); + scope->semantic_entered = result == 0; + } } out: errno = entry_errno; @@ -525,3 +539,15 @@ int latx_run_guest_callback(uintptr_t entry, const long *gpr_args, xmm_count, stack_args, stack_count, rax, rdx, xmm0, xmm1, st0, LATX_GUEST_USER_CALLBACK); } + +int latx_run_guest_callback_with_libc(uintptr_t entry, const long *gpr_args, + int gpr_count, const long *xmm_args, + int xmm_count, const long *stack_args, + int stack_count, long *rax, long *rdx, + long *xmm0, long *xmm1, + unsigned __int128 *st0) +{ + return run_guest_callback_impl(entry, gpr_args, gpr_count, xmm_args, + xmm_count, stack_args, stack_count, rax, rdx, xmm0, xmm1, st0, + LATX_GUEST_LIBC_CALLBACK); +} diff --git a/target/i386/latx/context/kzt-libc-semantic.c b/target/i386/latx/context/kzt-libc-semantic.c new file mode 100644 index 0000000000..8c95757d1d --- /dev/null +++ b/target/i386/latx/context/kzt-libc-semantic.c @@ -0,0 +1,1209 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "box64context.h" +#include "callback.h" +#include "debug.h" +#include "elfloader.h" +#include "kzt-guest-tls.h" +#include "kzt-libc-semantic.h" +#include "myalign.h" +#include "qemu/compiler.h" +#include "qemu.h" +#include "lsenv.h" + +#define KZT_LIBC_SEMANTIC_MAX_DEPTH 16 +#define KZT_LIBC_LOCALE_CATEGORIES 12 +#define KZT_LIBC_LOCALE_NAME_MAX 128 + +typedef enum kzt_libc_semantic_direction { + KZT_LIBC_HOST_TO_GUEST, + KZT_LIBC_GUEST_TO_HOST, +} kzt_libc_semantic_direction_t; + +typedef struct kzt_libc_semantic_frame { + kzt_libc_semantic_direction_t direction; +} kzt_libc_semantic_frame_t; + +typedef struct kzt_libc_locale_projection { + char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]; + uintptr_t guest_locale; + locale_t host_locale; + struct kzt_libc_locale_projection *next; +} kzt_libc_locale_projection_t; + +typedef struct kzt_libc_locale_state { + char current[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]; + char global[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]; + int uses_global; +} kzt_libc_locale_state_t; + +typedef struct kzt_libc_semantic_state { + uintptr_t guest_errno_slot; + uintptr_t guest_h_errno_slot; + /* + * Borrowed current-locale handles must never escape this context's + * lifetime. Cache entries are not shared with other contexts. + */ + kzt_libc_locale_projection_t *locale_projections; + kzt_libc_semantic_frame_t frames[KZT_LIBC_SEMANTIC_MAX_DEPTH]; + size_t depth; + size_t internal_depth; + size_t internal_overflow_depth; + int internal_guest_errno[KZT_LIBC_SEMANTIC_MAX_DEPTH]; + int internal_guest_errno_valid[KZT_LIBC_SEMANTIC_MAX_DEPTH]; + int internal_guest_h_errno[KZT_LIBC_SEMANTIC_MAX_DEPTH]; + int internal_guest_h_errno_valid[KZT_LIBC_SEMANTIC_MAX_DEPTH]; +} kzt_libc_semantic_state_t; + +static uintptr_t guest_errno_location; +static intptr_t guest_errno_offset; +static int guest_errno_offset_valid; +static uintptr_t guest_h_errno_location; +static intptr_t guest_h_errno_offset; +static int guest_h_errno_offset_valid; +static uintptr_t guest_newlocale; +static uintptr_t guest_uselocale; +static uintptr_t guest_freelocale; +static uintptr_t guest_duplocale; +static uintptr_t guest_nl_langinfo_l; +static uintptr_t guest_setlocale; +static GMutex kzt_libc_locale_projection_lock; +static GMutex kzt_libc_locale_sync_lock; +static int kzt_libc_semantic_required; +static GRecMutex kzt_libc_init_lock; +static gsize kzt_libc_init_lock_ready; + +static void kzt_libc_semantic_reinitialize_lock(GMutex *lock) +{ + memset(lock, 0, sizeof(*lock)); + g_mutex_init(lock); +} + +void kzt_libc_semantic_after_fork_child(void) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + if (kzt_libc_init_lock_ready) { + memset(&kzt_libc_init_lock, 0, sizeof(kzt_libc_init_lock)); + g_rec_mutex_init(&kzt_libc_init_lock); + } + kzt_libc_semantic_reinitialize_lock(&kzt_libc_locale_sync_lock); + kzt_libc_semantic_reinitialize_lock( + &kzt_libc_locale_projection_lock); +} + +static const int kzt_libc_locale_categories[KZT_LIBC_LOCALE_CATEGORIES] = { + LC_CTYPE, + LC_NUMERIC, + LC_TIME, + LC_COLLATE, + LC_MONETARY, + LC_MESSAGES, + LC_PAPER, + LC_NAME, + LC_ADDRESS, + LC_TELEPHONE, + LC_MEASUREMENT, + LC_IDENTIFICATION, +}; + +static const int kzt_libc_locale_masks[KZT_LIBC_LOCALE_CATEGORIES] = { + LC_CTYPE_MASK, + LC_NUMERIC_MASK, + LC_TIME_MASK, + LC_COLLATE_MASK, + LC_MONETARY_MASK, + LC_MESSAGES_MASK, + LC_PAPER_MASK, + LC_NAME_MASK, + LC_ADDRESS_MASK, + LC_TELEPHONE_MASK, + LC_MEASUREMENT_MASK, + LC_IDENTIFICATION_MASK, +}; + +/* POSIX pthread APIs return positive error numbers rather than -errno. */ +static void QEMU_NORETURN kzt_libc_semantic_abort_boundary( + const char *reason) +{ + fprintf(stderr, "KZT libc semantic boundary failed: %s\n", reason); + _exit(EXIT_FAILURE); +} + +static uintptr_t kzt_find_guest_libc_symbol(const char *name) +{ + if (!my_context || !name) { + return 0; + } + + for (int i = 0; i < my_context->elfsize; ++i) { + elfheader_t *head = my_context->elfs[i]; + const char *elf_name; + const char *base_name; + + if (!head) { + continue; + } + elf_name = ElfName(head); + if (!elf_name) { + continue; + } + base_name = strrchr(elf_name, '/'); + base_name = base_name ? base_name + 1 : elf_name; + if (strcmp(base_name, "libc.so.6") != 0 && + strncmp(base_name, "libc-", 5) != 0) { + continue; + } + return FindElfSymbolAddress(head, name); + } + return 0; +} + +static int kzt_libc_semantic_read_guest_errno( + const kzt_libc_semantic_state_t *state, int *value) +{ + int *slot; + + if (!state || !state->guest_errno_slot || !value) { + return -1; + } + slot = lock_user(VERIFY_READ, (abi_ulong)state->guest_errno_slot, + sizeof(*slot), 1); + if (!slot) { + return -1; + } + *value = *slot; + unlock_user(slot, (abi_ulong)state->guest_errno_slot, 0); + return 0; +} + +static int kzt_libc_semantic_write_guest_errno( + const kzt_libc_semantic_state_t *state, int value) +{ + int *slot; + + if (!state || !state->guest_errno_slot) { + return -1; + } + slot = lock_user(VERIFY_WRITE, (abi_ulong)state->guest_errno_slot, + sizeof(*slot), 0); + if (!slot) { + return -1; + } + *slot = value; + unlock_user(slot, (abi_ulong)state->guest_errno_slot, + sizeof(*slot)); + return 0; +} + +static int kzt_libc_semantic_read_guest_h_errno( + const kzt_libc_semantic_state_t *state, int *value) +{ + int *slot; + + if (!state || !state->guest_h_errno_slot || !value) { + return -1; + } + slot = lock_user(VERIFY_READ, (abi_ulong)state->guest_h_errno_slot, + sizeof(*slot), 1); + if (!slot) { + return -1; + } + *value = *slot; + unlock_user(slot, (abi_ulong)state->guest_h_errno_slot, 0); + return 0; +} + +static int kzt_libc_semantic_write_guest_h_errno( + const kzt_libc_semantic_state_t *state, int value) +{ + int *slot; + + if (!state || !state->guest_h_errno_slot) { + return -1; + } + slot = lock_user(VERIFY_WRITE, (abi_ulong)state->guest_h_errno_slot, + sizeof(*slot), 0); + if (!slot) { + return -1; + } + *slot = value; + unlock_user(slot, (abi_ulong)state->guest_h_errno_slot, + sizeof(*slot)); + return 0; +} + +static kzt_libc_semantic_state_t *kzt_libc_semantic_state( + CPUX86State *env) +{ + return env ? env->kzt_libc_semantic_state : NULL; +} + +static int kzt_libc_locale_names_equal( + const char left[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX], + const char right[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + if (strcmp(left[index], right[index]) != 0) { + return 0; + } + } + return 1; +} + +static int kzt_libc_capture_host_names( + locale_t locale, + char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + const char *name = nl_langinfo_l( + _NL_LOCALE_NAME(kzt_libc_locale_categories[index]), locale); + size_t length; + + if (!name) { + return -1; + } + length = strlen(name); + if (!length || length >= KZT_LIBC_LOCALE_NAME_MAX) { + return -1; + } + memcpy(names[index], name, length + 1); + } + return 0; +} + +static int kzt_libc_capture_host_locale_state( + kzt_libc_locale_state_t *state) +{ + locale_t current; + locale_t global; + int result = -1; + + if (!state) { + return -1; + } + current = uselocale((locale_t)0); + if (!current) { + return -1; + } + global = duplocale(LC_GLOBAL_LOCALE); + if (!global) { + return -1; + } + state->uses_global = current == LC_GLOBAL_LOCALE; + if (kzt_libc_capture_host_names(global, state->global) != 0) { + goto out; + } + if (state->uses_global) { + memcpy(state->current, state->global, + sizeof(state->current)); + } else if (kzt_libc_capture_host_names( + current, state->current) != 0) { + goto out; + } + result = 0; + +out: + freelocale(global); + return result; +} + +static int kzt_libc_read_guest_string( + uintptr_t address, char output[KZT_LIBC_LOCALE_NAME_MAX]) +{ + if (!address) { + return -1; + } + for (size_t index = 0; index < KZT_LIBC_LOCALE_NAME_MAX; ++index) { + char *byte = lock_user( + VERIFY_READ, (abi_ulong)(address + index), 1, 1); + + if (!byte) { + return -1; + } + output[index] = *byte; + unlock_user(byte, (abi_ulong)(address + index), 0); + if (!output[index]) { + return index ? 0 : -1; + } + } + return -1; +} + +static int kzt_libc_capture_guest_names( + uintptr_t locale, + char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + uintptr_t name = RunFunctionWithStateInternalNoRefresh( + guest_nl_langinfo_l, 2, + (uint64_t)_NL_LOCALE_NAME(kzt_libc_locale_categories[index]), + (uint64_t)locale); + + if (kzt_libc_read_guest_string(name, names[index]) != 0) { + return -1; + } + } + return 0; +} + +static int kzt_libc_capture_guest_locale_state( + kzt_libc_locale_state_t *state) +{ + uintptr_t current; + uintptr_t global; + int result = -1; + + if (!state) { + return -1; + } + current = RunFunctionWithStateInternalNoRefresh( + guest_uselocale, 1, 0); + if (!current) { + return -1; + } + global = RunFunctionWithStateInternalNoRefresh( + guest_duplocale, 1, (uint64_t)-1); + if (!global) { + return -1; + } + state->uses_global = current == UINTPTR_MAX; + if (kzt_libc_capture_guest_names(global, state->global) != 0) { + goto out; + } + if (state->uses_global) { + memcpy(state->current, state->global, + sizeof(state->current)); + } else if (kzt_libc_capture_guest_names( + current, state->current) != 0) { + goto out; + } + result = 0; + +out: + RunFunctionWithStateInternalNoRefresh( + guest_freelocale, 1, (uint64_t)global); + return result; +} + +static kzt_libc_locale_projection_t *kzt_libc_find_locale_projection( + kzt_libc_semantic_state_t *owner, + const char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + kzt_libc_locale_projection_t *projection; + + for (projection = owner->locale_projections; + projection; projection = projection->next) { + if (kzt_libc_locale_names_equal(projection->names, names)) { + return projection; + } + } + projection = g_new0(kzt_libc_locale_projection_t, 1); + memcpy(projection->names, names, sizeof(projection->names)); + projection->next = owner->locale_projections; + owner->locale_projections = projection; + return projection; +} + +static uintptr_t kzt_libc_get_guest_locale_projection( + kzt_libc_semantic_state_t *owner, + const char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + kzt_libc_locale_projection_t *projection; + uintptr_t locale; + + g_mutex_lock(&kzt_libc_locale_projection_lock); + projection = kzt_libc_find_locale_projection(owner, names); + if (!projection->guest_locale) { + locale = 0; + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + uintptr_t updated = RunFunctionWithStateInternalNoRefresh( + guest_newlocale, 3, + (uint64_t)kzt_libc_locale_masks[index], + (uint64_t)(uintptr_t)names[index], + (uint64_t)locale); + + if (!updated) { + if (locale) { + RunFunctionWithStateInternalNoRefresh( + guest_freelocale, 1, (uint64_t)locale); + } + locale = 0; + break; + } + locale = updated; + } + projection->guest_locale = locale; + } + locale = projection->guest_locale; + g_mutex_unlock(&kzt_libc_locale_projection_lock); + return locale; +} + +static locale_t kzt_libc_get_host_locale_projection( + kzt_libc_semantic_state_t *owner, + const char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + kzt_libc_locale_projection_t *projection; + locale_t locale; + + g_mutex_lock(&kzt_libc_locale_projection_lock); + projection = kzt_libc_find_locale_projection(owner, names); + if (!projection->host_locale) { + locale = (locale_t)0; + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + locale_t updated = newlocale( + kzt_libc_locale_masks[index], names[index], locale); + + if (!updated) { + if (locale) { + freelocale(locale); + } + locale = (locale_t)0; + break; + } + locale = updated; + } + projection->host_locale = locale; + } + locale = projection->host_locale; + g_mutex_unlock(&kzt_libc_locale_projection_lock); + return locale; +} + +static void kzt_libc_restore_host_global_locale( + const char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + (void)setlocale(kzt_libc_locale_categories[index], names[index]); + } +} + +static int kzt_libc_set_host_global_locale( + const char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + kzt_libc_locale_state_t previous; + + if (kzt_libc_capture_host_locale_state(&previous) != 0) { + return -1; + } + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + if (strcmp(previous.global[index], names[index]) != 0 && + !setlocale(kzt_libc_locale_categories[index], names[index])) { + kzt_libc_restore_host_global_locale(previous.global); + return -1; + } + } + return 0; +} + +static void kzt_libc_restore_guest_global_locale( + const char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + (void)RunFunctionWithStateInternalNoRefresh( + guest_setlocale, 2, + (uint64_t)kzt_libc_locale_categories[index], + (uint64_t)(uintptr_t)names[index]); + } +} + +static int kzt_libc_set_guest_global_locale( + const char names[KZT_LIBC_LOCALE_CATEGORIES][KZT_LIBC_LOCALE_NAME_MAX]) +{ + kzt_libc_locale_state_t previous; + + if (kzt_libc_capture_guest_locale_state(&previous) != 0) { + return -1; + } + for (size_t index = 0; + index < KZT_LIBC_LOCALE_CATEGORIES; ++index) { + if (strcmp(previous.global[index], names[index]) != 0 && + !RunFunctionWithStateInternalNoRefresh( + guest_setlocale, 2, + (uint64_t)kzt_libc_locale_categories[index], + (uint64_t)(uintptr_t)names[index])) { + kzt_libc_restore_guest_global_locale(previous.global); + return -1; + } + } + return 0; +} + +static int kzt_libc_install_host_locale_state( + kzt_libc_semantic_state_t *owner, + const kzt_libc_locale_state_t *state) +{ + locale_t locale; + + if (!state || kzt_libc_set_host_global_locale(state->global) != 0) { + return -1; + } + locale = state->uses_global + ? LC_GLOBAL_LOCALE + : kzt_libc_get_host_locale_projection(owner, state->current); + if (!locale || !uselocale(locale)) { + return -1; + } + return 0; +} + +static int kzt_libc_install_guest_locale_state( + kzt_libc_semantic_state_t *owner, + const kzt_libc_locale_state_t *state) +{ + uintptr_t locale; + + if (!state || kzt_libc_set_guest_global_locale(state->global) != 0) { + return -1; + } + locale = state->uses_global + ? UINTPTR_MAX + : kzt_libc_get_guest_locale_projection(owner, state->current); + if (!locale || !RunFunctionWithStateInternalNoRefresh( + guest_uselocale, 1, (uint64_t)locale)) { + return -1; + } + return 0; +} + +static int kzt_libc_prepare_process_locale(kzt_libc_semantic_state_t *owner) +{ + kzt_libc_locale_state_t guest_state; + kzt_libc_locale_state_t host_state; + int result = -1; + + g_mutex_lock(&kzt_libc_locale_sync_lock); + if (kzt_libc_capture_host_locale_state(&host_state) != 0 || + kzt_libc_capture_guest_locale_state(&guest_state) != 0) { + goto out; + } + if ((!host_state.uses_global && + !kzt_libc_get_guest_locale_projection(owner, host_state.current)) || + (!guest_state.uses_global && + !kzt_libc_get_host_locale_projection(owner, guest_state.current))) { + goto out; + } + result = 0; + +out: + g_mutex_unlock(&kzt_libc_locale_sync_lock); + return result; +} + +static int kzt_libc_semantic_initialize_locked(CPUX86State *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return -1; + } + kzt_libc_semantic_state_t *state; + uintptr_t h_errno_slot; + uintptr_t slot; + + if (!env) { + return -1; + } + if (env->kzt_libc_semantic_state) { + return 0; + } + if (!guest_newlocale) { + guest_newlocale = kzt_find_guest_libc_symbol("newlocale"); + guest_uselocale = kzt_find_guest_libc_symbol("uselocale"); + guest_freelocale = kzt_find_guest_libc_symbol("freelocale"); + guest_duplocale = kzt_find_guest_libc_symbol("duplocale"); + guest_nl_langinfo_l = kzt_find_guest_libc_symbol("nl_langinfo_l"); + guest_setlocale = kzt_find_guest_libc_symbol("setlocale"); + if (!guest_newlocale) { + guest_newlocale = kzt_resolve_guest_symbol("newlocale"); + } + if (!guest_uselocale) { + guest_uselocale = kzt_resolve_guest_symbol("uselocale"); + } + if (!guest_freelocale) { + guest_freelocale = kzt_resolve_guest_symbol("freelocale"); + } + if (!guest_duplocale) { + guest_duplocale = kzt_resolve_guest_symbol("duplocale"); + } + if (!guest_nl_langinfo_l) { + guest_nl_langinfo_l = kzt_resolve_guest_symbol("nl_langinfo_l"); + } + if (!guest_setlocale) { + guest_setlocale = kzt_resolve_guest_symbol("setlocale"); + } + } + if (!guest_newlocale || !guest_uselocale || !guest_freelocale || + !guest_duplocale || !guest_nl_langinfo_l || !guest_setlocale) { + printf_log(LOG_INFO, + "KZT cannot resolve Guest locale projection helpers\n"); + return -1; + } + if (!guest_errno_offset_valid) { + if (!guest_errno_location) { + guest_errno_location = kzt_find_guest_libc_symbol( + "__errno_location"); + if (!guest_errno_location) { + guest_errno_location = kzt_resolve_guest_symbol( + "__errno_location"); + } + } + if (!guest_errno_location) { + printf_log(LOG_INFO, + "KZT cannot resolve Guest __errno_location\n"); + return -1; + } + slot = RunFunctionWithStateInternal(guest_errno_location, 0); + if (!slot || !env->segs[R_FS].base) { + return -1; + } + guest_errno_offset = (intptr_t)slot - + (intptr_t)env->segs[R_FS].base; + guest_errno_offset_valid = 1; + } else { + if (!env->segs[R_FS].base || + (guest_errno_offset > 0 && + env->segs[R_FS].base > + UINTPTR_MAX - (uintptr_t)guest_errno_offset) || + (guest_errno_offset < 0 && + env->segs[R_FS].base < + (uintptr_t)-guest_errno_offset)) { + return -1; + } + slot = (uintptr_t)((intptr_t)env->segs[R_FS].base + + guest_errno_offset); + } + if (!guest_h_errno_offset_valid) { + if (!guest_h_errno_location) { + guest_h_errno_location = kzt_find_guest_libc_symbol( + "__h_errno_location"); + if (!guest_h_errno_location) { + guest_h_errno_location = kzt_resolve_guest_symbol( + "__h_errno_location"); + } + } + if (!guest_h_errno_location) { + printf_log(LOG_INFO, + "KZT cannot resolve Guest __h_errno_location\n"); + return -1; + } + h_errno_slot = RunFunctionWithStateInternal( + guest_h_errno_location, 0); + if (!h_errno_slot || !env->segs[R_FS].base) { + return -1; + } + guest_h_errno_offset = (intptr_t)h_errno_slot - + (intptr_t)env->segs[R_FS].base; + guest_h_errno_offset_valid = 1; + } else { + if (!env->segs[R_FS].base || + (guest_h_errno_offset > 0 && + env->segs[R_FS].base > + UINTPTR_MAX - (uintptr_t)guest_h_errno_offset) || + (guest_h_errno_offset < 0 && + env->segs[R_FS].base < + (uintptr_t)-guest_h_errno_offset)) { + return -1; + } + h_errno_slot = + (uintptr_t)((intptr_t)env->segs[R_FS].base + + guest_h_errno_offset); + } + state = g_new0(kzt_libc_semantic_state_t, 1); + state->guest_errno_slot = slot; + state->guest_h_errno_slot = h_errno_slot; + env->kzt_libc_semantic_state = state; + return 0; +} + +int kzt_libc_semantic_initialize(CPUX86State *env) +{ + int result; + + if (!latx_kzt_guest_tls_enabled()) { + return -1; + } + if (g_once_init_enter(&kzt_libc_init_lock_ready)) { + g_rec_mutex_init(&kzt_libc_init_lock); + g_once_init_leave(&kzt_libc_init_lock_ready, 1); + } + g_rec_mutex_lock(&kzt_libc_init_lock); + result = kzt_libc_semantic_initialize_locked(env); + g_rec_mutex_unlock(&kzt_libc_init_lock); + return result; +} + +int kzt_libc_semantic_enter_current(void) +{ + CPUX86State *env; + + if (!latx_kzt_guest_tls_enabled() || !lsenv || !lsenv->cpu_state) { + return -1; + } + env = (CPUX86State *)lsenv->cpu_state; + if (kzt_libc_semantic_initialize(env) != 0 || + (!kzt_libc_semantic_process_ready() && + kzt_libc_semantic_prepare_process_locale(env) != 0)) { + return -1; + } + kzt_libc_semantic_guest_to_host_enter(env); + return 0; +} + +void kzt_libc_semantic_leave_current(void) +{ + if (lsenv && lsenv->cpu_state) { + kzt_libc_semantic_guest_to_host_leave( + (CPUX86State *)lsenv->cpu_state); + } +} + +int kzt_libc_semantic_process_ready(void) +{ + return g_atomic_int_get(&kzt_libc_semantic_required); +} + +static void kzt_libc_release_locale_projections( + kzt_libc_semantic_state_t *state, bool guest_available) +{ + kzt_libc_locale_projection_t *projection; + + if (!state) { + return; + } + projection = state->locale_projections; + state->locale_projections = NULL; + while (projection) { + kzt_libc_locale_projection_t *next = projection->next; + + if (projection->host_locale) { + if (uselocale((locale_t)0) == projection->host_locale) { + uselocale(LC_GLOBAL_LOCALE); + } + freelocale(projection->host_locale); + } + if (guest_available && projection->guest_locale) { + if (RunFunctionWithStateInternalNoRefresh(guest_uselocale, 1, 0) + == projection->guest_locale) { + RunFunctionWithStateInternalNoRefresh( + guest_uselocale, 1, (uint64_t)-1); + } + RunFunctionWithStateInternalNoRefresh( + guest_freelocale, 1, projection->guest_locale); + } + g_free(projection); + projection = next; + } +} + +void kzt_libc_semantic_process_reset(CPUX86State *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return; + } + if (env) { + /* A replacement Guest image cannot execute the old allocator. */ + kzt_libc_release_locale_projections(env->kzt_libc_semantic_state, + false); + g_free(env->kzt_libc_semantic_state); + env->kzt_libc_semantic_state = NULL; + } + guest_errno_location = 0; + guest_errno_offset = 0; + guest_errno_offset_valid = 0; + guest_h_errno_location = 0; + guest_h_errno_offset = 0; + guest_h_errno_offset_valid = 0; + guest_newlocale = 0; + guest_uselocale = 0; + guest_freelocale = 0; + guest_duplocale = 0; + guest_nl_langinfo_l = 0; + guest_setlocale = 0; + g_atomic_int_set(&kzt_libc_semantic_required, 0); + +} + +int kzt_libc_semantic_prepare_process_locale(CPUX86State *env) +{ + if (!latx_kzt_guest_tls_enabled()) { + return -1; + } + int result; + + if (!env || !env->kzt_libc_semantic_state) { + return -1; + } + result = kzt_libc_prepare_process_locale(env->kzt_libc_semantic_state); + if (result == 0) { + g_atomic_int_set(&kzt_libc_semantic_required, 1); + } + return result; +} + +uintptr_t kzt_libc_semantic_setlocale(CPUX86State *env, int category, + const char *locale) +{ + kzt_libc_locale_state_t previous; + kzt_libc_locale_state_t updated; + uintptr_t guest_result; + + if (!env || !env->kzt_libc_semantic_state || !guest_setlocale) { + return 0; + } + if (!locale) { + return RunFunctionWithStateInternalNoRefresh( + guest_setlocale, 2, (uint64_t)category, 0); + } + + g_mutex_lock(&kzt_libc_locale_sync_lock); + if (kzt_libc_capture_guest_locale_state(&previous) != 0) { + g_mutex_unlock(&kzt_libc_locale_sync_lock); + return 0; + } + guest_result = RunFunctionWithStateInternalNoRefresh( + guest_setlocale, 2, (uint64_t)category, + (uint64_t)(uintptr_t)locale); + if (!guest_result || + kzt_libc_capture_guest_locale_state(&updated) != 0 || + kzt_libc_set_host_global_locale(updated.global) != 0) { + kzt_libc_restore_guest_global_locale(previous.global); + guest_result = 0; + } + g_mutex_unlock(&kzt_libc_locale_sync_lock); + return guest_result; +} + +void kzt_libc_semantic_destroy(CPUX86State *env) +{ + if (env) { + kzt_libc_release_locale_projections(env->kzt_libc_semantic_state, true); + g_free(env->kzt_libc_semantic_state); + env->kzt_libc_semantic_state = NULL; + } +} + +void kzt_libc_semantic_internal_enter(CPUX86State *env) +{ + kzt_libc_semantic_state_t *state = kzt_libc_semantic_state(env); + + if (!state) { + return; + } + if (state->internal_depth == KZT_LIBC_SEMANTIC_MAX_DEPTH) { + if (state->internal_overflow_depth != SIZE_MAX) { + ++state->internal_overflow_depth; + } + return; + } + { + size_t depth = state->internal_depth++; + + state->internal_guest_errno_valid[depth] = + kzt_libc_semantic_read_guest_errno( + state, &state->internal_guest_errno[depth]) == 0; + state->internal_guest_h_errno_valid[depth] = + kzt_libc_semantic_read_guest_h_errno( + state, &state->internal_guest_h_errno[depth]) == 0; + } +} + +void kzt_libc_semantic_internal_leave(CPUX86State *env) +{ + kzt_libc_semantic_state_t *state = kzt_libc_semantic_state(env); + + if (state && state->internal_overflow_depth) { + --state->internal_overflow_depth; + return; + } + if (state && state->internal_depth) { + size_t depth = --state->internal_depth; + + if (state->internal_guest_errno_valid[depth]) { + (void)kzt_libc_semantic_write_guest_errno( + state, state->internal_guest_errno[depth]); + state->internal_guest_errno_valid[depth] = 0; + } + if (state->internal_guest_h_errno_valid[depth]) { + (void)kzt_libc_semantic_write_guest_h_errno( + state, state->internal_guest_h_errno[depth]); + state->internal_guest_h_errno_valid[depth] = 0; + } + } +} + +static kzt_libc_semantic_frame_t *kzt_libc_semantic_push( + kzt_libc_semantic_state_t *state, + kzt_libc_semantic_direction_t direction) +{ + kzt_libc_semantic_frame_t *frame; + + if (!state || state->internal_depth || state->internal_overflow_depth || + state->depth == KZT_LIBC_SEMANTIC_MAX_DEPTH) { + return NULL; + } + frame = &state->frames[state->depth++]; + memset(frame, 0, sizeof(*frame)); + frame->direction = direction; + return frame; +} + +static int kzt_libc_semantic_pop( + kzt_libc_semantic_state_t *state, + kzt_libc_semantic_direction_t direction) +{ + if (!state || !state->depth || + state->frames[state->depth - 1].direction != direction) { + return -1; + } + --state->depth; + return 0; +} + +int kzt_libc_semantic_host_to_guest_enter(CPUX86State *env, + int host_errno, + int host_h_errno) +{ + kzt_libc_semantic_state_t *state = kzt_libc_semantic_state(env); + kzt_libc_locale_state_t locale_state; + int locale_result; + + if (!kzt_libc_semantic_process_ready()) { + return 0; + } + if (!state) { + if (kzt_libc_semantic_initialize(env) != 0) { + return -1; + } + state = kzt_libc_semantic_state(env); + } + if (!kzt_libc_semantic_push(state, KZT_LIBC_HOST_TO_GUEST)) { + return -1; + } + g_mutex_lock(&kzt_libc_locale_sync_lock); + locale_result = + kzt_libc_capture_host_locale_state(&locale_state) == 0 && + kzt_libc_install_guest_locale_state(state, &locale_state) == 0 + ? 0 : -1; + g_mutex_unlock(&kzt_libc_locale_sync_lock); + if (locale_result != 0) { + --state->depth; + return -1; + } + if (kzt_libc_semantic_write_guest_errno( + state, host_to_target_errno(host_errno)) != 0) { + --state->depth; + return -1; + } + if (kzt_libc_semantic_write_guest_h_errno( + state, host_h_errno) != 0) { + --state->depth; + return -1; + } + return 0; +} + +void kzt_libc_semantic_host_to_guest_leave(CPUX86State *env) +{ + kzt_libc_semantic_state_t *state = kzt_libc_semantic_state(env); + kzt_libc_locale_state_t locale_state; + int guest_errno = 0; + int guest_h_errno = 0; + int host_errno; + int have_guest_errno; + int have_guest_h_errno; + int locale_result; + + if (!kzt_libc_semantic_process_ready()) { + return; + } + if (!state || !state->depth || + state->frames[state->depth - 1].direction != + KZT_LIBC_HOST_TO_GUEST) { + return; + } + have_guest_errno = + kzt_libc_semantic_read_guest_errno(state, &guest_errno) == 0; + have_guest_h_errno = + kzt_libc_semantic_read_guest_h_errno( + state, &guest_h_errno) == 0; + g_mutex_lock(&kzt_libc_locale_sync_lock); + locale_result = + kzt_libc_capture_guest_locale_state(&locale_state) == 0 && + kzt_libc_install_host_locale_state(state, &locale_state) == 0 + ? 0 : -1; + g_mutex_unlock(&kzt_libc_locale_sync_lock); + if (locale_result != 0) { + kzt_libc_semantic_abort_boundary( + "cannot propagate Guest locale on callback return"); + } + if (kzt_libc_semantic_pop( + state, KZT_LIBC_HOST_TO_GUEST) != 0) { + kzt_libc_semantic_abort_boundary( + "unbalanced Host-to-Guest return"); + } + if (!have_guest_errno) { + kzt_libc_semantic_abort_boundary( + "cannot read Guest errno on callback return"); + } + if (!have_guest_h_errno) { + kzt_libc_semantic_abort_boundary( + "cannot read Guest h_errno on callback return"); + } + host_errno = target_to_host_errno(guest_errno); + errno = host_errno; + h_errno = guest_h_errno; +} + +void kzt_libc_semantic_guest_to_host_enter(CPUX86State *env) +{ + kzt_libc_semantic_state_t *state = kzt_libc_semantic_state(env); + kzt_libc_locale_state_t locale_state; + int guest_errno; + int guest_h_errno; + int locale_result; + + if (!kzt_libc_semantic_process_ready()) { + return; + } + if (!state) { + /* + * Attached Host threads initialize Guest TLS before semantic + * state. Guest libc bootstrap helpers may cross a wrapped + * symbol in that narrow interval, but no user callback can run + * until both initializers succeed. + */ + if (env && env->kzt_guest_tls_allocation) { + return; + } + if (kzt_libc_semantic_initialize(env) != 0) { + kzt_libc_semantic_abort_boundary( + "cannot initialize per-thread state on Guest-to-Host entry"); + } + state = kzt_libc_semantic_state(env); + } + if (state->internal_depth || state->internal_overflow_depth) { + return; + } + if (kzt_libc_semantic_read_guest_errno(state, &guest_errno) != 0) { + kzt_libc_semantic_abort_boundary( + "cannot read Guest errno on Guest-to-Host entry"); + } + if (kzt_libc_semantic_read_guest_h_errno( + state, &guest_h_errno) != 0) { + kzt_libc_semantic_abort_boundary( + "cannot read Guest h_errno on Guest-to-Host entry"); + } + if (!kzt_libc_semantic_push(state, KZT_LIBC_GUEST_TO_HOST)) { + kzt_libc_semantic_abort_boundary( + "Guest-to-Host call-frame depth exceeded"); + } + g_mutex_lock(&kzt_libc_locale_sync_lock); + locale_result = + kzt_libc_capture_guest_locale_state(&locale_state) == 0 && + kzt_libc_install_host_locale_state(state, &locale_state) == 0 + ? 0 : -1; + g_mutex_unlock(&kzt_libc_locale_sync_lock); + if (locale_result != 0) { + kzt_libc_semantic_abort_boundary( + "cannot propagate Guest locale on Guest-to-Host entry"); + } + errno = target_to_host_errno(guest_errno); + h_errno = guest_h_errno; +} + +void kzt_libc_semantic_guest_to_host_leave(CPUX86State *env) +{ + kzt_libc_semantic_state_t *state = kzt_libc_semantic_state(env); + kzt_libc_locale_state_t locale_state; + unsigned char guest_fp_return[sizeof(env->fpregs[0])]; + unsigned char guest_xmm0_return[sizeof(env->xmm_regs[0])]; + unsigned char guest_xmm1_return[sizeof(env->xmm_regs[1])]; + uint64_t guest_rax; + uint64_t guest_rdx; + int guest_fpstt; + int host_errno = errno; + int host_h_errno = h_errno; + int locale_result; + + if (!kzt_libc_semantic_process_ready()) { + return; + } + if (!state) { + if (env && env->kzt_guest_tls_allocation) { + return; + } + return; + } + if (!state->depth || + state->frames[state->depth - 1].direction != + KZT_LIBC_GUEST_TO_HOST) { + return; + } + if (state->internal_depth || state->internal_overflow_depth) { + return; + } + guest_rax = env->regs[R_EAX]; + guest_rdx = env->regs[R_EDX]; + guest_fpstt = env->fpstt; + memcpy(guest_xmm0_return, &env->xmm_regs[0], + sizeof(guest_xmm0_return)); + memcpy(guest_xmm1_return, &env->xmm_regs[1], + sizeof(guest_xmm1_return)); + memcpy(guest_fp_return, &env->fpregs[guest_fpstt & 7], + sizeof(guest_fp_return)); + g_mutex_lock(&kzt_libc_locale_sync_lock); + locale_result = + kzt_libc_capture_host_locale_state(&locale_state) == 0 && + kzt_libc_install_guest_locale_state(state, &locale_state) == 0 + ? 0 : -1; + g_mutex_unlock(&kzt_libc_locale_sync_lock); + if (locale_result != 0) { + kzt_libc_semantic_abort_boundary( + "cannot propagate Host locale on Guest-to-Host return"); + } + if (kzt_libc_semantic_pop(state, KZT_LIBC_GUEST_TO_HOST) != 0) { + kzt_libc_semantic_abort_boundary( + "unbalanced Guest-to-Host return"); + } + env->regs[R_EAX] = guest_rax; + env->regs[R_EDX] = guest_rdx; + env->fpstt = guest_fpstt; + memcpy(&env->xmm_regs[0], guest_xmm0_return, + sizeof(guest_xmm0_return)); + memcpy(&env->xmm_regs[1], guest_xmm1_return, + sizeof(guest_xmm1_return)); + memcpy(&env->fpregs[guest_fpstt & 7], guest_fp_return, + sizeof(guest_fp_return)); + if (kzt_libc_semantic_write_guest_errno( + state, host_to_target_errno(host_errno)) != 0) { + kzt_libc_semantic_abort_boundary( + "cannot write Guest errno on Guest-to-Host return"); + } + if (kzt_libc_semantic_write_guest_h_errno( + state, host_h_errno) != 0) { + kzt_libc_semantic_abort_boundary( + "cannot write Guest h_errno on Guest-to-Host return"); + } + errno = host_errno; + h_errno = host_h_errno; +} diff --git a/target/i386/latx/context/meson.build b/target/i386/latx/context/meson.build index ad850e98c5..133ab0a183 100644 --- a/target/i386/latx/context/meson.build +++ b/target/i386/latx/context/meson.build @@ -99,3 +99,4 @@ my_file = files( i386_ss.add(when: 'CONFIG_LATX', if_true: my_file) i386_ss.add(when: 'CONFIG_LATX_KZT', if_true: files('kzt-guest-tls.c')) i386_ss.add(when: 'CONFIG_LATX_KZT', if_true: files('kzt-guest-thread.c')) +i386_ss.add(when: 'CONFIG_LATX_KZT', if_true: files('kzt-libc-semantic.c')) diff --git a/target/i386/latx/context/myalign.c b/target/i386/latx/context/myalign.c index a128269dd3..9a976db5e9 100644 --- a/target/i386/latx/context/myalign.c +++ b/target/i386/latx/context/myalign.c @@ -17,6 +17,7 @@ #include "kzt-groups.h" #include "kzt_public_loader_observer.h" #include "kzt-guest-tls.h" +#include "kzt-libc-semantic.h" #include "kzt_relro_preprotect.h" #include "latx-options.h" #include "librarian_private.h" @@ -3763,6 +3764,7 @@ static void kzt_guest_main_entry_callback(CPUX86State *env) uintptr_t dynamic_addr = 0; size_t dynamic_count = 0; + kzt_libc_semantic_process_reset(env); kzt_guest_tls_loader_tracking_reset(); if (latx_finalize_host_thread_template(env) != 0) { fprintf(stderr, diff --git a/target/i386/latx/context/wrappedlibc.c b/target/i386/latx/context/wrappedlibc.c index bcafdd150c..c4415c4078 100644 --- a/target/i386/latx/context/wrappedlibc.c +++ b/target/i386/latx/context/wrappedlibc.c @@ -70,6 +70,7 @@ #include "bridge.h" #include "globalsymbols.h" #include "x86dlfun.h" +#include "kzt-libc-semantic.h" #include "kzt-guest-tls.h" #include "kzt-guest-thread.h" @@ -3381,7 +3382,13 @@ EXPORT int my_register_printf_type(void* f) return my->register_printf_type(findprintf_typeFct(f)); } +EXPORT void *my_setlocale(int category, const char *locale) +{ + __MY_CPU; + return (void *)kzt_libc_semantic_setlocale( + cpu, category, locale); +} EXPORT int my_pthread_key_create(unsigned int *key, void *destructor) { diff --git a/target/i386/latx/include/callback.h b/target/i386/latx/include/callback.h index e64fed103a..b0af9471c0 100644 --- a/target/i386/latx/include/callback.h +++ b/target/i386/latx/include/callback.h @@ -20,6 +20,11 @@ int latx_run_guest_callback(uintptr_t entry, const long *gpr_args, long *xmm0, long *xmm1, unsigned __int128 *st0); +int latx_run_guest_callback_with_libc( + uintptr_t entry, const long *gpr_args, int gpr_count, + const long *xmm_args, int xmm_count, const long *stack_args, + int stack_count, long *rax, long *rdx, long *xmm0, long *xmm1, + unsigned __int128 *st0); #endif //__CALLBACK_H__ diff --git a/target/i386/latx/include/kzt-libc-semantic.h b/target/i386/latx/include/kzt-libc-semantic.h new file mode 100644 index 0000000000..044740f3ad --- /dev/null +++ b/target/i386/latx/include/kzt-libc-semantic.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef LATX_KZT_LIBC_SEMANTIC_H +#define LATX_KZT_LIBC_SEMANTIC_H + +#include +#include + +typedef struct CPUX86State CPUX86State; + +#ifdef CONFIG_LATX_KZT +/* + * Non-global locale projections are owned by this Guest context. Handles + * observed through uselocale(0) are borrowed until context destruction: + * use duplocale() before passing them to freelocale() or as newlocale() base. + * Never share such a borrowed handle with another thread/context. + */ +int kzt_libc_semantic_enter_current(void); +void kzt_libc_semantic_leave_current(void); +int kzt_libc_semantic_initialize(CPUX86State *env); +int kzt_libc_semantic_process_ready(void); +void kzt_libc_semantic_process_reset(CPUX86State *env); +void kzt_libc_semantic_after_fork_child(void); +int kzt_libc_semantic_prepare_process_locale(CPUX86State *env); +void kzt_libc_semantic_destroy(CPUX86State *env); + +void kzt_libc_semantic_internal_enter(CPUX86State *env); +void kzt_libc_semantic_internal_leave(CPUX86State *env); + +int kzt_libc_semantic_host_to_guest_enter(CPUX86State *env, + int host_errno, + int host_h_errno); +void kzt_libc_semantic_host_to_guest_leave(CPUX86State *env); + +void kzt_libc_semantic_guest_to_host_enter(CPUX86State *env); +void kzt_libc_semantic_guest_to_host_leave(CPUX86State *env); + +uintptr_t kzt_libc_semantic_setlocale(CPUX86State *env, int category, + const char *locale); + +#else +static inline int kzt_libc_semantic_initialize(CPUX86State *env) +{ + (void)env; + return 0; +} + +static inline int kzt_libc_semantic_process_ready(void) +{ + return 0; +} + +static inline void kzt_libc_semantic_process_reset(CPUX86State *env) +{ + (void)env; +} + +static inline void kzt_libc_semantic_after_fork_child(void) +{ +} + +static inline void kzt_libc_semantic_destroy(CPUX86State *env) +{ + (void)env; +} + +static inline int kzt_libc_semantic_prepare_process_locale( + CPUX86State *env) +{ + (void)env; + return 0; +} + +static inline void kzt_libc_semantic_internal_enter(CPUX86State *env) +{ + (void)env; +} + +static inline void kzt_libc_semantic_internal_leave(CPUX86State *env) +{ + (void)env; +} + +static inline int kzt_libc_semantic_host_to_guest_enter( + CPUX86State *env, int host_errno, int host_h_errno) +{ + (void)env; + (void)host_errno; + (void)host_h_errno; + return 0; +} + +static inline void kzt_libc_semantic_host_to_guest_leave(CPUX86State *env) +{ + (void)env; +} + +static inline void kzt_libc_semantic_guest_to_host_enter(CPUX86State *env) +{ + (void)env; +} + +static inline void kzt_libc_semantic_guest_to_host_leave(CPUX86State *env) +{ + (void)env; +} + +static inline uintptr_t kzt_libc_semantic_setlocale( + CPUX86State *env, int category, const char *locale) +{ + (void)env; + (void)category; + (void)locale; + return 0; +} + +#endif + +#endif diff --git a/tests/integration/kzt-libc-boundary.c b/tests/integration/kzt-libc-boundary.c new file mode 100644 index 0000000000..83f28b92fa --- /dev/null +++ b/tests/integration/kzt-libc-boundary.c @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HOST_PROBE +typedef int (*GuestCall)(uintptr_t, const long *, int, const long *, int, + const long *, int, long *, long *, long *, long *, + unsigned __int128 *); +typedef struct Worker { + Display *display; + int result; + int index; +} Worker; + +static void *callback_thread(void *opaque) +{ + Worker *worker = opaque; + _XAsyncHandler *handler = worker->display->async_handlers; + xReply reply = { 0 }; + locale_t locale = newlocale(LC_ALL_MASK, "C.UTF-8", NULL); + locale_t previous; + long result = 0; + GuestCall invoke = (GuestCall)dlsym(RTLD_DEFAULT, + "latx_run_guest_callback_with_libc"); + const long args[] = { + (long)(uintptr_t)worker->display, (long)(uintptr_t)&reply, + 0, worker->index * 10, (long)(uintptr_t)handler->data, + }; + int status; + + if (!invoke || !handler->data) { + if (locale) { + freelocale(locale); + } + worker->result = 1; + return NULL; + } + if (!locale) { + locale = newlocale(LC_ALL_MASK, "C.utf8", NULL); + } + if (!locale) { + worker->result = 1; + return NULL; + } + previous = uselocale(locale); + errno = ENOENT; + h_errno = NO_RECOVERY; + status = invoke(*(uintptr_t *)handler->data, args, 5, NULL, 0, NULL, 0, + &result, NULL, NULL, NULL, NULL); + worker->result = status || !result || errno != EACCES || + h_errno != NO_DATA || MB_CUR_MAX != 1; + uselocale(previous); + freelocale(locale); + return NULL; +} + +int XEventsQueued(Display *display, int mode) +{ + int (*enter)(void) = dlsym(RTLD_DEFAULT, "kzt_libc_semantic_enter_current"); + void (*leave)(void) = dlsym(RTLD_DEFAULT, "kzt_libc_semantic_leave_current"); + Worker workers[2] = { { display, 0, 0 }, { display, 0, 1 } }; + pthread_t threads[2]; + int result = 0; + (void)mode; + + if (!enter || !leave || enter() != 0) { + return 1; + } + if (errno != EAGAIN || h_errno != HOST_NOT_FOUND || MB_CUR_MAX <= 1) { + result = 2; + } else if (pthread_create(&threads[0], NULL, callback_thread, &workers[0])) { + result = 3; + } else { + if (pthread_create(&threads[1], NULL, callback_thread, &workers[1])) { + result = 3; + } else { + pthread_join(threads[1], NULL); + } + pthread_join(threads[0], NULL); + if (!result && (workers[0].result || workers[1].result)) { + result = 4; + } + } + if (!result) { + xReply reply = { 0 }; + _XAsyncHandler *handler = display->async_handlers; + + errno = ENOENT; + h_errno = NO_RECOVERY; + if (!handler->handler(display, &reply, NULL, 1, handler->data) || + errno != ENOENT || h_errno != NO_RECOVERY) { + result = 5; + } + } + errno = EIO; + h_errno = TRY_AGAIN; + leave(); + return result; +} +#else +static int calls; +static pthread_barrier_t projection_barrier; +static locale_t projected[2]; + +static Bool callback(Display *display, xReply *reply, char *buffer, + int length, XPointer opaque) +{ + int valid = errno == ENOENT && h_errno == NO_RECOVERY && MB_CUR_MAX > 1; + (void)display; + (void)reply; + (void)buffer; + (void)length; + (void)opaque; + + if (length == 1) { + valid = errno == EAGAIN && h_errno == HOST_NOT_FOUND && + MB_CUR_MAX > 1; + errno = ESRCH; + h_errno = NO_DATA; + __atomic_add_fetch(&calls, 1, __ATOMIC_RELAXED); + return valid; + } + projected[length / 10] = uselocale((locale_t)0); + pthread_barrier_wait(&projection_barrier); + if (projected[0] == projected[1]) { + fprintf(stderr, "FAIL: attached contexts share a locale projection\n"); + valid = 0; + } + /* A current projection is borrowed. Mutate/free only an owned copy. */ + locale_t owned = duplocale(projected[length / 10]); + if (owned) { + locale_t changed = newlocale(LC_NUMERIC_MASK, "C", owned); + + if (changed) { + owned = changed; + } else { + valid = 0; + } + freelocale(owned); + } else { + valid = 0; + } + if (MB_CUR_MAX <= 1) { + valid = 0; + } + if (!uselocale(LC_GLOBAL_LOCALE) || MB_CUR_MAX != 1) { + valid = 0; + } + __atomic_add_fetch(&calls, 1, __ATOMIC_RELAXED); + errno = EACCES; + h_errno = NO_DATA; + return valid; +} + +int main(void) +{ + Display *display = calloc(1, sizeof(*display)); + _XAsyncHandler handler = { .handler = callback }; + uintptr_t callback_entry = (uintptr_t)callback; + locale_t locale = newlocale(LC_ALL_MASK, "C.UTF-8", NULL); + locale_t previous; + int result; + + if (!locale) { + locale = newlocale(LC_ALL_MASK, "C.utf8", NULL); + } + if (!display || !locale) { + return 2; + } + previous = uselocale(locale); + if (pthread_barrier_init(&projection_barrier, NULL, 2)) { + return 4; + } + handler.data = (XPointer)&callback_entry; + display->async_handlers = &handler; + errno = EAGAIN; + h_errno = HOST_NOT_FOUND; + result = XEventsQueued(display, 0); + if (result || calls != 3 || errno != EIO || h_errno != TRY_AGAIN || + MB_CUR_MAX <= 1) { + fprintf(stderr, "FAIL: boundary result=%d calls=%d errno=%d h_errno=%d\n", + result, calls, errno, h_errno); + return 3; + } + uselocale(previous); + freelocale(locale); + pthread_barrier_destroy(&projection_barrier); + free(display); + puts("PASS: explicit bidirectional errno, h_errno and locale boundaries"); + return 0; +} +#endif diff --git a/tests/integration/registrations/x11-kzt/meson.build b/tests/integration/registrations/x11-kzt/meson.build index 27d4ab66c7..65b8ad47b6 100644 --- a/tests/integration/registrations/x11-kzt/meson.build +++ b/tests/integration/registrations/x11-kzt/meson.build @@ -88,6 +88,16 @@ if 'x86_64-linux-user' in target_dirs and \ 'timeout': 180, }] + latx_integration_tests += [{ + 'name': 'test-kzt-libc-boundary', + 'runner': find_program('../../test-kzt-libc-boundary.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../kzt-libc-boundary.c'), + files('../../x11-async-bridge-dummy.c'), + ], + 'timeout': 120, + }] foreach mode : ['dtv', 'read', 'futex', 'seccomp', 'seccomp-restart-code', diff --git a/tests/integration/test-kzt-libc-boundary.sh b/tests/integration/test-kzt-libc-boundary.sh new file mode 100755 index 0000000000..fc50f1fd1c --- /dev/null +++ b/tests/integration/test-kzt-libc-boundary.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later +set -eu +emulator=$1 +source_file=$2 +dummy_source=$3 +guest_root=${LATX_X86_64_SYSROOT:-/usr/gnemul/latx-x86_64} +guest_cc=${LATX_X86_64_CC:-x86_64-linux-gnu-gcc} +native_cc=${LATX_NATIVE_CC:-cc} +if [ "$(uname -m)" != loongarch64 ] || + ! command -v "$guest_cc" >/dev/null 2>&1; then + echo "SKIP: requires LoongArch and a Guest C compiler" + exit 77 +fi +task_dir=$(mktemp -d) +trap 'rm -rf "$task_dir"' EXIT HUP INT TERM +mkdir -p "$task_dir/guest" +"$native_cc" -shared -fPIC -O2 -Wall -Wextra -Werror -DHOST_PROBE \ + "$source_file" -pthread -ldl -Wl,-soname,libX11.so.6 \ + -o "$task_dir/libX11.so.6" +"$guest_cc" --sysroot="$guest_root" -shared -fPIC \ + -I"$(dirname "$dummy_source")" "$dummy_source" \ + -Wl,-soname,libX11.so.6 -o "$task_dir/guest/libX11.so.6" +"$guest_cc" --sysroot="$guest_root" -O2 -Wall -Wextra -Werror \ + "$source_file" -L"$task_dir/guest" -l:libX11.so.6 \ + -Wl,--no-as-needed -ldl -Wl,--as-needed -pthread \ + -o "$task_dir/guest/boundary" +LD_PRELOAD="$task_dir/libX11.so.6" \ +BOX64_LD_LIBRARY_PATH="$task_dir/guest" \ +LATX_AOT=0 LATX_KZT=1 LATX_KZT_LIBS=core,x11 LATX_KZT_GUEST_TLS=1 \ + "$emulator" -U LD_PRELOAD -E "LD_LIBRARY_PATH=$task_dir/guest" \ + -L "$guest_root" "$task_dir/guest/boundary"