From f62182baa46a736003b5f8ab5d8a100421c7c632 Mon Sep 17 00:00:00 2001 From: yuerengan Date: Wed, 2 Sep 2026 11:15:46 +0800 Subject: [PATCH 1/3] LATX: merge PUSH/POP stack pointer updates during translation Track a pending stack pointer delta while lowering consecutive register or immediate PUSH and register POP instructions. Fold the delta into stack accesses and materialize it at stack-sensitive instructions and TB boundaries. Store the pending delta in restore metadata so faults and signals expose the precise guest stack pointer. Keep the existing IR2 optimizer as a mutually exclusive compile-time rollback path. Add x86_64 and i386 integration coverage for widths, stack aliases, helper boundaries, protected-page faults, signal context restore, file mappings, and legacy stack instructions. Signed-off-by: yuerengan --- accel/tcg/translate-all.c | 12 + target/i386/latx/include/env.h | 3 + target/i386/latx/include/optimize-config.h | 7 +- target/i386/latx/include/translate.h | 4 + .../i386/latx/optimization/ir2-optimization.c | 5 +- target/i386/latx/translator/tr-mov.c | 125 ++++-- target/i386/latx/translator/translate.c | 19 + target/i386/tcg/translate.c | 4 + tests/integration/push-pop-sp-file.S | 45 ++ tests/integration/push-pop-sp-i386.S | 174 ++++++++ tests/integration/push-pop-sp.S | 404 ++++++++++++++++++ .../registrations/process/meson.build | 17 + tests/integration/test-push-pop-sp-i386.sh | 47 ++ tests/integration/test-push-pop-sp.sh | 68 +++ 14 files changed, 908 insertions(+), 26 deletions(-) create mode 100644 tests/integration/push-pop-sp-file.S create mode 100644 tests/integration/push-pop-sp-i386.S create mode 100644 tests/integration/push-pop-sp.S create mode 100755 tests/integration/test-push-pop-sp-i386.sh create mode 100755 tests/integration/test-push-pop-sp.sh diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index f4d515e257a..3432bad2c7a 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -474,6 +474,18 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, cpu_neg(cpu)->icount_decr.u16.low += num_insns - i; } restore_state_to_opc(env, tb, data); +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + target_long sp_delta = (target_long)data[1]; +#ifdef TARGET_X86_64 + if (tb->bool_flags & IS_CODE64) { + env->regs[R_ESP] += sp_delta; + } else { + env->regs[R_ESP] = (uint32_t)(env->regs[R_ESP] + sp_delta); + } +#else + env->regs[R_ESP] += sp_delta; +#endif +#endif #ifdef CONFIG_LATX_OPT_PUSH_POP uint32_t parallel = cpu->tcg_cflags & CF_PARALLEL; if (!parallel) { diff --git a/target/i386/latx/include/env.h b/target/i386/latx/include/env.h index c1afc6a431f..ad016073a7d 100644 --- a/target/i386/latx/include/env.h +++ b/target/i386/latx/include/env.h @@ -26,6 +26,9 @@ typedef struct TRANSLATION_DATA { /* ir1 */ IR1_INST *curr_ir1_inst; int curr_ir1_count; +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + int sp_delta; +#endif /* uint8 ir1_dump_threshold[MAX_IR1_NUM_PER_TB]; */ /* ir2 */ diff --git a/target/i386/latx/include/optimize-config.h b/target/i386/latx/include/optimize-config.h index 34b5d294050..1f5fa4a932e 100644 --- a/target/i386/latx/include/optimize-config.h +++ b/target/i386/latx/include/optimize-config.h @@ -78,7 +78,12 @@ #undef CONFIG_LATX_JRRA #define CONFIG_LATX_JRRA /* jr-ra, */ #undef CONFIG_LATX_OPT_PUSH_POP -#define CONFIG_LATX_OPT_PUSH_POP +#undef CONFIG_LATX_OPT_PUSH_POP_TRANS +#define CONFIG_LATX_OPT_PUSH_POP_TRANS +#if defined(CONFIG_LATX_OPT_PUSH_POP) && \ + defined(CONFIG_LATX_OPT_PUSH_POP_TRANS) +#error "PUSH/POP optimizations must be mutually exclusive" +#endif #undef CONFIG_LATX_SSSE3_SSE4 #define CONFIG_LATX_SSSE3_SSE4 #undef CONFIG_LATX_SPLIT_TB diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 38200bc4298..3a19a698933 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1502,6 +1502,10 @@ void tr_disasm(struct TranslationBlock *tb, int max_insns); void etb_add_succ(void* etb,int depth); int tr_translate_tb(struct TranslationBlock *tb); int tr_ir2_generate(struct TranslationBlock *tb); +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS +bool latx_sp_merge_can_delay(IR1_INST *pir1); +void latx_sp_merge_flush(void); +#endif int label_dispose(TranslationBlock *tb, TRANSLATION_DATA *lat_ctx); int tr_ir2_assemble(const void *code_start_addr, const IR2_INST *pir2); #if defined(CONFIG_LATX_FLAG_REDUCTION) && \ diff --git a/target/i386/latx/optimization/ir2-optimization.c b/target/i386/latx/optimization/ir2-optimization.c index 58288b5a9ba..eb8302ab990 100644 --- a/target/i386/latx/optimization/ir2-optimization.c +++ b/target/i386/latx/optimization/ir2-optimization.c @@ -215,6 +215,7 @@ void tri_avoid_leading_label(void) #endif } +#ifdef CONFIG_LATX_OPT_PUSH_POP static int ir2_get_addi_rsp_offs(IR2_INST *ir2) { int rsp = reg_gpr_map[esp_index]; @@ -435,7 +436,6 @@ static void ir2_opt_push_pop(TranslationBlock *tb) } IR2_INST *curr; int curr_id, end_id, off, patch_off; - //int branch_id = 0; bool insert; IR2_OPCODE op; @@ -525,6 +525,7 @@ static void ir2_opt_push_pop(TranslationBlock *tb) "patch_off = %-5d\n", curr_id, 0); } } +#endif void tr_ir2_optimize(TranslationBlock *tb) { @@ -545,6 +546,7 @@ void tr_ir2_optimize(TranslationBlock *tb) } */ } +#ifdef CONFIG_LATX_OPT_PUSH_POP extern void ir1_optimization(TranslationBlock *tb); void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i) @@ -704,3 +706,4 @@ void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i) env->regs[R_ESP] += patch_off; tr_fini(false); } +#endif diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index 42597187389..dab5886138a 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -12,6 +12,44 @@ #include "translate.h" #include "hbr.h" #include "latx-smc.h" + +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS +bool latx_sp_merge_can_delay(IR1_INST *pir1) +{ + IR1_OPCODE opc = ir1_opcode(pir1); + IR1_OPND *opnd0; + + if (opc != dt_X86_INS_PUSH && opc != dt_X86_INS_POP) { + return false; + } + opnd0 = ir1_get_opnd(pir1, 0); + + if (ir1_opnd_is_gpr_used(opnd0, esp_index)) { + return false; + } + if (opc == dt_X86_INS_PUSH) { + return ir1_opnd_is_gpr(opnd0) || ir1_opnd_is_imm(opnd0); + } + return opc == dt_X86_INS_POP && ir1_opnd_is_gpr(opnd0); +} + +void latx_sp_merge_flush(void) +{ + TRANSLATION_DATA *td = lsenv->tr_data; + + if (td->sp_delta) { + IR2_OPND esp_opnd = ra_alloc_gpr(esp_index); + + la_addi_addrx(esp_opnd, esp_opnd, td->sp_delta); +#ifdef CONFIG_LATX_DEBUG + qemu_log_mask(LAT_IR2_SCHED, + "[LAT_PUSH_TRANS] flush delta=%d\n", td->sp_delta); +#endif + td->sp_delta = 0; + } +} +#endif + bool translate_pop(IR1_INST *pir1) { IR2_OPND esp_opnd = ra_alloc_gpr(esp_index); @@ -19,6 +57,13 @@ bool translate_pop(IR1_INST *pir1) int has_esp = ir1_opnd_is_gpr_used(ir1_get_opnd(pir1, 0), esp_index); int pop_size = ir1_opnd_size(ir1_get_opnd(pir1, 0)); int esp_increment = pop_size >> 3; +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + bool delay_sp = latx_sp_merge_can_delay(pir1); + int sp_delta = delay_sp ? lsenv->tr_data->sp_delta : 0; +#else + bool delay_sp = false; + int sp_delta = 0; +#endif if (pop_size == 16 && ir1_opnd_is_seg(ir1_get_opnd(pir1, 0))) { #ifdef TARGET_X86_64 @@ -36,13 +81,13 @@ bool translate_pop(IR1_INST *pir1) #ifdef TARGET_X86_64 if (CODEIS64) { if (pop_size == 64) {/* 64 bits */ - ir1_opnd_build_mem(&mem_ir1_opnd, 64, dt_X86_REG_RSP, 0); + ir1_opnd_build_mem(&mem_ir1_opnd, 64, dt_X86_REG_RSP, sp_delta); if (ir1_opnd_is_gpr(ir1_get_opnd(pir1, 0))) { /* when dest is gpr, load into gpr directly */ IR2_OPND dest_opnd = load_ireg_from_ir1(ir1_get_opnd(pir1, 0), UNKNOWN_EXTENSION, false); load_ireg_from_ir1_2(dest_opnd, &mem_ir1_opnd, UNKNOWN_EXTENSION, false); - if (!has_esp) { + if (!has_esp && !delay_sp) { la_addi_addrx(esp_opnd, esp_opnd, esp_increment); } @@ -59,7 +104,8 @@ bool translate_pop(IR1_INST *pir1) esp_increment); } } else if (pop_size == 16) { - ir1_opnd_build_mem(&mem_ir1_opnd, 16, dt_X86_REG_ESP, 0); + ir1_opnd_build_mem(&mem_ir1_opnd, 16, dt_X86_REG_ESP, + sp_delta); /* load value */ IR2_OPND value_opnd = load_ireg_from_ir1(&mem_ir1_opnd, ZERO_EXTENSION, false); @@ -68,10 +114,11 @@ bool translate_pop(IR1_INST *pir1) dest_ir1_opnd->mem.disp += esp_increment; } store_ireg_to_ir1(value_opnd, dest_ir1_opnd, false); - la_addi_addrx(esp_opnd, esp_opnd, - esp_increment); + if (!delay_sp) { + la_addi_addrx(esp_opnd, esp_opnd, esp_increment); + } } else { - ir1_opnd_build_mem(&mem_ir1_opnd, 32, dt_X86_REG_ESP, 0); + ir1_opnd_build_mem(&mem_ir1_opnd, 32, dt_X86_REG_ESP, sp_delta); if (ir1_opnd_is_gpr(ir1_get_opnd(pir1, 0))) { IR2_OPND dest_opnd = load_ireg_from_ir1(ir1_get_opnd(pir1, 0), UNKNOWN_EXTENSION, false); @@ -80,7 +127,7 @@ bool translate_pop(IR1_INST *pir1) // dest_em = ZERO_EXTENSION; load_ireg_from_ir1_2(dest_opnd, &mem_ir1_opnd, dest_em, false); - if (!has_esp) { + if (!has_esp && !delay_sp) { la_addi_addrx(esp_opnd, esp_opnd, esp_increment); } @@ -99,7 +146,7 @@ bool translate_pop(IR1_INST *pir1) } } else { if (pop_size == 16) { - ir1_opnd_build_mem(&mem_ir1_opnd, 16, dt_X86_REG_ESP, 0); + ir1_opnd_build_mem(&mem_ir1_opnd, 16, dt_X86_REG_ESP, sp_delta); /* load value */ IR2_OPND value_opnd = load_ireg_from_ir1(&mem_ir1_opnd, ZERO_EXTENSION, false); @@ -108,10 +155,11 @@ bool translate_pop(IR1_INST *pir1) dest_ir1_opnd->mem.disp += esp_increment; } store_ireg_to_ir1(value_opnd, dest_ir1_opnd, false); - la_addi_addrx(esp_opnd, esp_opnd, - esp_increment); + if (!delay_sp) { + la_addi_addrx(esp_opnd, esp_opnd, esp_increment); + } } else { - ir1_opnd_build_mem(&mem_ir1_opnd, 32, dt_X86_REG_ESP, 0); + ir1_opnd_build_mem(&mem_ir1_opnd, 32, dt_X86_REG_ESP, sp_delta); if (ir1_opnd_is_gpr(ir1_get_opnd(pir1, 0))) { IR2_OPND dest_opnd = load_ireg_from_ir1(ir1_get_opnd(pir1, 0), UNKNOWN_EXTENSION, false); @@ -120,7 +168,7 @@ bool translate_pop(IR1_INST *pir1) // dest_em = ZERO_EXTENSION; load_ireg_from_ir1_2(dest_opnd, &mem_ir1_opnd, dest_em, false); - if (!has_esp) { + if (!has_esp && !delay_sp) { la_addi_addrx(esp_opnd, esp_opnd, esp_increment); } @@ -133,14 +181,15 @@ bool translate_pop(IR1_INST *pir1) dest_ir1_opnd->mem.disp += esp_increment; } store_ireg_to_ir1(value_opnd, dest_ir1_opnd, false); - la_addi_addrx(esp_opnd, esp_opnd, - esp_increment); + if (!delay_sp) { + la_addi_addrx(esp_opnd, esp_opnd, esp_increment); + } } } } #else if (pop_size == 16) { - ir1_opnd_build_mem(&mem_ir1_opnd, 16, dt_X86_REG_ESP, 0); + ir1_opnd_build_mem(&mem_ir1_opnd, 16, dt_X86_REG_ESP, sp_delta); /* load value */ IR2_OPND value_opnd = load_ireg_from_ir1(&mem_ir1_opnd, ZERO_EXTENSION, false); @@ -149,10 +198,11 @@ bool translate_pop(IR1_INST *pir1) dest_ir1_opnd->mem.disp += esp_increment; } store_ireg_to_ir1(value_opnd, dest_ir1_opnd, false); - la_addi_addrx(esp_opnd, esp_opnd, - esp_increment); + if (!delay_sp) { + la_addi_addrx(esp_opnd, esp_opnd, esp_increment); + } } else { - ir1_opnd_build_mem(&mem_ir1_opnd, 32, dt_X86_REG_ESP, 0); + ir1_opnd_build_mem(&mem_ir1_opnd, 32, dt_X86_REG_ESP, sp_delta); if (ir1_opnd_is_gpr(ir1_get_opnd(pir1, 0))) { IR2_OPND dest_opnd = load_ireg_from_ir1(ir1_get_opnd(pir1, 0), UNKNOWN_EXTENSION, false); @@ -161,7 +211,7 @@ bool translate_pop(IR1_INST *pir1) // dest_em = ZERO_EXTENSION; load_ireg_from_ir1_2(dest_opnd, &mem_ir1_opnd, dest_em, false); - if (!has_esp) { + if (!has_esp && !delay_sp) { la_addi_addrx(esp_opnd, esp_opnd, esp_increment); } @@ -178,6 +228,16 @@ bool translate_pop(IR1_INST *pir1) esp_increment); } } +#endif +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + if (delay_sp) { + lsenv->tr_data->sp_delta += esp_increment; +#ifdef CONFIG_LATX_DEBUG + qemu_log_mask(LAT_IR2_SCHED, + "[LAT_PUSH_TRANS] pop delta=%d\n", + lsenv->tr_data->sp_delta); +#endif + } #endif return true; } @@ -187,6 +247,12 @@ bool translate_push(IR1_INST *pir1) IR2_OPND esp_opnd = ra_alloc_gpr(esp_index); int push_size = ir1_opnd_size(ir1_get_opnd(pir1, 0)); int esp_decrement = push_size >> 3; +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + bool delay_sp = latx_sp_merge_can_delay(pir1); + int sp_delta = delay_sp ? lsenv->tr_data->sp_delta : 0; +#else + int sp_delta = 0; +#endif if (push_size == 16 && ir1_opnd_is_seg(ir1_get_opnd(pir1, 0))) { #ifdef TARGET_X86_64 @@ -205,22 +271,33 @@ bool translate_push(IR1_INST *pir1) #ifndef TARGET_X86_64 ir1_opnd_build_mem(&mem_ir1_opnd, esp_decrement << 3, - dt_X86_REG_ESP, -esp_decrement); + dt_X86_REG_ESP, sp_delta - esp_decrement); #else if (CODEIS64) { ir1_opnd_build_mem(&mem_ir1_opnd, esp_decrement << 3, - dt_X86_REG_RSP, -esp_decrement); + dt_X86_REG_RSP, sp_delta - esp_decrement); } else { ir1_opnd_build_mem(&mem_ir1_opnd, esp_decrement << 3, - dt_X86_REG_ESP, -esp_decrement); + dt_X86_REG_ESP, sp_delta - esp_decrement); } #endif IR2_OPND value_opnd = load_ireg_from_ir1(ir1_get_opnd(pir1, 0), UNKNOWN_EXTENSION, false); store_ireg_to_ir1(value_opnd, &mem_ir1_opnd, false); - la_addi_addrx(esp_opnd, esp_opnd, - -esp_decrement); +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + if (delay_sp) { + lsenv->tr_data->sp_delta -= esp_decrement; +#ifdef CONFIG_LATX_DEBUG + qemu_log_mask(LAT_IR2_SCHED, + "[LAT_PUSH_TRANS] push delta=%d\n", + lsenv->tr_data->sp_delta); +#endif + } else +#endif + { + la_addi_addrx(esp_opnd, esp_opnd, -esp_decrement); + } return true; } diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index f491539813b..f5512184f97 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -107,6 +107,9 @@ void tr_init(void *tb) t->curr_tb); t->curr_tb = tb; t->curr_ir1_inst = NULL; +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + t->sp_delta = 0; +#endif /* register allocation init */ ra_free_all(); @@ -147,6 +150,9 @@ void tr_fini(bool check_the_extension) /* set current tb and ir1 */ t->curr_tb = NULL; t->curr_ir1_inst = NULL; +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + t->sp_delta = 0; +#endif /* reset ir2 array */ t->ir2_inst_num_current = 0; @@ -2198,12 +2204,22 @@ int tr_ir2_generate(struct TranslationBlock *tb) } #endif for (i = 0; i < ir1_nr; ++i) { +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + if (!latx_sp_merge_can_delay(pir1)) { + latx_sp_merge_flush(); + } +#endif /* * handle segv scenario, store host pc to gen_insn_data and encode to a BYTE * at the end of TB translate cache. */ tcg_ctx->gen_insn_data[i][0] = pir1->info->address; +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + tcg_ctx->gen_insn_data[i][1] = + (target_ulong)(target_long)lsenv->tr_data->sp_delta; +#else tcg_ctx->gen_insn_data[i][1] = 0; +#endif #ifdef CONFIG_LATX_IMM_REG imm_cache->curr_ir1_index = i; @@ -2260,6 +2276,9 @@ int tr_ir2_generate(struct TranslationBlock *tb) pir1++; } +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + latx_sp_merge_flush(); +#endif #ifdef CONFIG_LATX_DEBUG if (option_dump_ir1) { pir1 = tb_ir1_inst(tb, 0); diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 002afd5e414..ab52484a3a9 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -8752,7 +8752,11 @@ static void restore_extcontext(CPUX86State *env, ucontext_t *uc) void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, target_ulong *data) { +#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS + int cc_op = CC_OP_DYNAMIC; +#else int cc_op = data[1]; +#endif /* env->eip = data[0] - tb->cs_base; */ env->eip = data[0]; diff --git a/tests/integration/push-pop-sp-file.S b/tests/integration/push-pop-sp-file.S new file mode 100644 index 00000000000..6b5cd88692b --- /dev/null +++ b/tests/integration/push-pop-sp-file.S @@ -0,0 +1,45 @@ +.section .text +.global file_mapped_push_pop +.type file_mapped_push_pop, @function +file_mapped_push_pop: + mov %rsp, %r11 + mov $0x11, %rax + mov $0x22, %rbx + mov $0x33, %rcx + mov $0x44, %rdx + push %rax + push %rbx + push %rcx + push %rdx + pop %r8 + pop %r9 + pop %r10 + pop %rsi + cmp $0x44, %r8 + jne .Lfailed + cmp $0x33, %r9 + jne .Lfailed + cmp $0x22, %r10 + jne .Lfailed + cmp $0x11, %rsi + jne .Lfailed + cmp %r11, %rsp + jne .Lfailed + lea file_mapped_fault(%rip), %rdx + xor %eax, %eax + ret +.Lfailed: + mov $40, %eax + ret +.size file_mapped_push_pop, .-file_mapped_push_pop + +.type file_mapped_fault, @function +file_mapped_fault: + push %rax + push %rbx + xor %eax, %eax + mov (%rax), %rax + ud2 +.size file_mapped_fault, .-file_mapped_fault + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/push-pop-sp-i386.S b/tests/integration/push-pop-sp-i386.S new file mode 100644 index 00000000000..7038c45aec5 --- /dev/null +++ b/tests/integration/push-pop-sp-i386.S @@ -0,0 +1,174 @@ +.equ __NR_exit, 1 +.equ __NR_getpid, 20 +.equ __NR_rt_sigreturn, 173 +.equ __NR_rt_sigaction, 174 +.equ SIGSEGV, 11 +.equ SA_SIGINFO, 0x00000004 +.equ SA_RESTORER, 0x04000000 +.equ UCONTEXT_ESP_OFFSET, 48 +.equ UCONTEXT_EIP_OFFSET, 76 + +.section .data +.align 4 +segv_action: + .long segv_handler + .long (SA_SIGINFO | SA_RESTORER) + .long signal_restorer + .quad 0 +saved_esp: + .long 0 +expected_esp: + .long 0 +fault_resume: + .long 0 +fault_seen: + .long 0 + +.section .text +.global _start +.type _start, @function +_start: + mov $__NR_rt_sigaction, %eax + mov $SIGSEGV, %ebx + mov $segv_action, %ecx + xor %edx, %edx + mov $8, %esi + int $0x80 + test %eax, %eax + js .Linstall_failed + + call test_push_pop_32 + mov %eax, %ebx + mov $__NR_exit, %eax + int $0x80 +.Linstall_failed: + mov $10, %ebx + mov $__NR_exit, %eax + int $0x80 + +.type test_push_pop_32, @function +test_push_pop_32: + mov %esp, saved_esp + mov $0x11, %eax + mov $0x22, %ebx + mov $0x33, %ecx + mov $0x44, %edx + push %eax + push %ebx + push %ecx + push %edx + pop %esi + pop %edi + pop %ebp + pop %eax + cmp $0x44, %esi + jne .Lpush_pop_failed + cmp $0x33, %edi + jne .Lpush_pop_failed + cmp $0x22, %ebp + jne .Lpush_pop_failed + cmp $0x11, %eax + jne .Lpush_pop_failed + cmp saved_esp, %esp + jne .Lpush_pop_failed + + mov %esp, %eax + push %esp + pop %ecx + cmp %eax, %ecx + jne .Lpush_esp_failed + cmp saved_esp, %esp + jne .Lpush_esp_failed + + pushal + popal + pushfl + pop %eax + push %eax + popfl + enter $32, $0 + leave + cmp saved_esp, %esp + jne .Lstack_instructions_failed + + push %eax + push %ebx + mov $__NR_getpid, %eax + int $0x80 + test %eax, %eax + js .Lhelper_failed + pop %ebx + pop %eax + cmp saved_esp, %esp + jne .Lhelper_failed + + mov %esp, %eax + sub $8, %eax + mov %eax, expected_esp + movl $.Lafter_fault, fault_resume + movl $0, fault_seen + push %esi + push %edi + xor %eax, %eax + mov (%eax), %eax +.Lafter_fault: + cmpl $1, fault_seen + jne .Lfault_not_seen + pop %edi + pop %esi + cmp saved_esp, %esp + jne .Lfault_restore_failed + + xor %eax, %eax + ret +.Lpush_pop_failed: + mov $20, %eax + ret +.Lpush_esp_failed: + mov $21, %eax + ret +.Lstack_instructions_failed: + mov $22, %eax + ret +.Lhelper_failed: + mov $23, %eax + ret +.Lfault_not_seen: + mov $24, %eax + ret +.Lfault_restore_failed: + mov $25, %eax + ret +.size test_push_pop_32, .-test_push_pop_32 + +.type segv_handler, @function +segv_handler: + cmpl $SIGSEGV, 4(%esp) + jne .Lwrong_signal + mov 12(%esp), %edx + mov expected_esp, %eax + cmp %eax, UCONTEXT_ESP_OFFSET(%edx) + jne .Lwrong_fault_esp + mov fault_resume, %eax + mov %eax, UCONTEXT_EIP_OFFSET(%edx) + movl $1, fault_seen + ret +.size segv_handler, .-segv_handler + +.Lwrong_signal: + mov $30, %ebx + jmp .Lexit_now +.Lwrong_fault_esp: + mov $31, %ebx +.Lexit_now: + mov $__NR_exit, %eax + int $0x80 + +.type signal_restorer, @function +signal_restorer: + mov $__NR_rt_sigreturn, %eax + int $0x80 + ud2 +.size signal_restorer, .-signal_restorer + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/push-pop-sp.S b/tests/integration/push-pop-sp.S new file mode 100644 index 00000000000..ba9073a43ce --- /dev/null +++ b/tests/integration/push-pop-sp.S @@ -0,0 +1,404 @@ +.equ __NR_rt_sigaction, 13 +.equ __NR_rt_sigreturn, 15 +.equ __NR_mmap, 9 +.equ __NR_mprotect, 10 +.equ __NR_munmap, 11 +.equ __NR_close, 3 +.equ __NR_getpid, 39 +.equ __NR_exit, 60 +.equ __NR_openat, 257 +.equ __NR_sigaltstack, 131 +.equ SIGSEGV, 11 +.equ SA_SIGINFO, 0x00000004 +.equ SA_ONSTACK, 0x08000000 +.equ SA_RESTORER, 0x04000000 +.equ AT_FDCWD, -100 +.equ PROT_NONE, 0 +.equ PROT_READ_WRITE, 3 +.equ PROT_READ_EXEC, 5 +.equ MAP_PRIVATE, 2 +.equ MAP_PRIVATE_ANONYMOUS, 0x22 +.equ UCONTEXT_RSP_OFFSET, 160 +.equ UCONTEXT_RIP_OFFSET, 168 + +.section .data +.align 8 +segv_action: + .quad segv_handler + .quad (SA_SIGINFO | SA_ONSTACK | SA_RESTORER) + .quad signal_restorer + .quad 0 +altstack: + .quad 0 + .quad 0 + .quad 65536 +expected_rsp: + .quad 0 +fault_resume: + .quad 0 +resume_rsp: + .quad 0 +fault_seen: + .long 0 +.align 8 +file_code_path: + .quad 0 + +.section .text +.global _start +.type _start, @function +_start: + cmpq $2, (%rsp) + jb .Largs_ready + mov 16(%rsp), %rax + mov %rax, file_code_path(%rip) +.Largs_ready: + mov $__NR_mmap, %eax + xor %edi, %edi + mov $65536, %esi + mov $PROT_READ_WRITE, %edx + mov $MAP_PRIVATE_ANONYMOUS, %r10d + mov $-1, %r8 + xor %r9d, %r9d + syscall + test %rax, %rax + js altstack_failed + mov %rax, altstack(%rip) + mov $__NR_sigaltstack, %eax + lea altstack(%rip), %rdi + xor %esi, %esi + syscall + test %rax, %rax + js altstack_failed + + mov $__NR_rt_sigaction, %eax + mov $SIGSEGV, %edi + lea segv_action(%rip), %rsi + xor %edx, %edx + mov $8, %r10d + syscall + test %rax, %rax + js install_failed + + call test_push_pop + test %eax, %eax + jnz exit + call test_rsp_uses + test %eax, %eax + jnz exit + call test_stack_instructions + test %eax, %eax + jnz exit + call test_helper_boundary + test %eax, %eax + jnz exit + call test_fault_restore + test %eax, %eax + jnz exit + call test_push_guard_fault + test %eax, %eax + jnz exit + call test_file_mapping + test %eax, %eax + jnz exit + + xor %edi, %edi + jmp do_exit + +install_failed: + mov $10, %edi + jmp do_exit +altstack_failed: + mov $11, %edi + jmp do_exit + +.type test_push_pop, @function +test_push_pop: + mov %rsp, %r11 + mov $0x11, %rax + mov $0x22, %rbx + mov $0x33, %rcx + mov $0x44, %rdx + push %rax + push %rbx + push %rcx + push %rdx + pop %r8 + pop %r9 + pop %r10 + pop %rsi + cmp $0x44, %r8 + jne .Lpush_pop_failed + cmp $0x33, %r9 + jne .Lpush_pop_failed + cmp $0x22, %r10 + jne .Lpush_pop_failed + cmp $0x11, %rsi + jne .Lpush_pop_failed + cmp %r11, %rsp + jne .Lpush_pop_failed + + mov $0x1122, %eax + mov $0x3344, %ebx + pushw %ax + pushw %bx + popw %cx + popw %dx + cmp $0x3344, %cx + jne .Lpush_pop_failed + cmp $0x1122, %dx + jne .Lpush_pop_failed + cmp %r11, %rsp + jne .Lpush_pop_failed + xor %eax, %eax + ret +.Lpush_pop_failed: + mov $20, %eax + ret +.size test_push_pop, .-test_push_pop + +.type test_rsp_uses, @function +test_rsp_uses: + mov %rsp, %r11 + movabs $0x1122334455667788, %rax + push %rax + cmp %rax, (%rsp) + jne .Lrsp_uses_failed + mov %sp, %sp + mov %spl, %spl + pop %rcx + cmp %rax, %rcx + jne .Lrsp_uses_failed + cmp %r11, %rsp + jne .Lrsp_uses_failed + xor %eax, %eax + ret +.Lrsp_uses_failed: + mov $21, %eax + ret +.size test_rsp_uses, .-test_rsp_uses + +.type test_stack_instructions, @function +test_stack_instructions: + mov %rsp, %r11 + pushfq + pop %rax + push %rax + popfq + enter $32, $0 + leave + cmp %r11, %rsp + jne .Lstack_instructions_failed + xor %eax, %eax + ret +.Lstack_instructions_failed: + mov $22, %eax + ret +.size test_stack_instructions, .-test_stack_instructions + +.type test_helper_boundary, @function +test_helper_boundary: + mov %rsp, %r12 + push %rax + push %rbx + mov $__NR_getpid, %eax + syscall + test %rax, %rax + js .Lhelper_boundary_failed + pop %rbx + pop %rax + cmp %r12, %rsp + jne .Lhelper_boundary_failed + xor %eax, %eax + ret +.Lhelper_boundary_failed: + mov $23, %eax + ret +.size test_helper_boundary, .-test_helper_boundary + +.type test_fault_restore, @function +test_fault_restore: + mov %rsp, %r11 + lea -16(%r11), %rax + mov %rax, expected_rsp(%rip) + mov %rax, resume_rsp(%rip) + lea .Lafter_fault(%rip), %rax + mov %rax, fault_resume(%rip) + movl $0, fault_seen(%rip) + push %r12 + push %r13 + xor %eax, %eax + mov (%rax), %rax +.Lafter_fault: + cmpl $1, fault_seen(%rip) + jne .Lfault_not_seen + pop %r13 + pop %r12 + cmp %r11, %rsp + jne .Lfault_restore_failed + xor %eax, %eax + ret +.Lfault_not_seen: + mov $24, %eax + ret +.Lfault_restore_failed: + mov $25, %eax + ret +.size test_fault_restore, .-test_fault_restore + +.type test_push_guard_fault, @function +test_push_guard_fault: + mov %rsp, %r15 + mov $__NR_mmap, %eax + xor %edi, %edi + mov $8192, %esi + mov $PROT_READ_WRITE, %edx + mov $MAP_PRIVATE_ANONYMOUS, %r10d + mov $-1, %r8 + xor %r9d, %r9d + syscall + test %rax, %rax + js .Lguard_setup_failed + mov %rax, %r14 + + mov $__NR_mprotect, %eax + mov %r14, %rdi + mov $4096, %esi + mov $PROT_NONE, %edx + syscall + test %rax, %rax + js .Lguard_unmap_failed + + lea 4096(%r14), %rax + mov %rax, expected_rsp(%rip) + mov %r15, resume_rsp(%rip) + lea .Lafter_guard_fault(%rip), %rax + mov %rax, fault_resume(%rip) + movl $0, fault_seen(%rip) + lea 4104(%r14), %rsp + push %r12 + push %r13 +.Lafter_guard_fault: + cmpl $1, fault_seen(%rip) + jne .Lguard_fault_not_seen + mov $__NR_munmap, %eax + mov %r14, %rdi + mov $8192, %esi + syscall + xor %eax, %eax + ret + +.Lguard_unmap_failed: + mov $__NR_munmap, %eax + mov %r14, %rdi + mov $8192, %esi + syscall +.Lguard_setup_failed: + mov $29, %eax + ret +.Lguard_fault_not_seen: + mov $28, %eax + ret +.size test_push_guard_fault, .-test_push_guard_fault + +.type test_file_mapping, @function +test_file_mapping: + mov file_code_path(%rip), %rsi + test %rsi, %rsi + jz .Lfile_mapping_failed + mov $__NR_openat, %eax + mov $AT_FDCWD, %edi + xor %edx, %edx + xor %r10d, %r10d + syscall + test %rax, %rax + js .Lfile_mapping_failed + mov %rax, %r14 + + mov $__NR_mmap, %eax + xor %edi, %edi + mov $4096, %esi + mov $PROT_READ_EXEC, %edx + mov $MAP_PRIVATE, %r10d + mov %r14, %r8 + xor %r9d, %r9d + syscall + test %rax, %rax + js .Lclose_mapping_fd + + call *%rax + mov %eax, %r13d + mov %rdx, %r12 + mov $__NR_close, %eax + mov %r14, %rdi + syscall + test %r13d, %r13d + jnz .Lfile_mapping_result + test %r12, %r12 + jz .Lfile_mapping_failed + + lea -24(%rsp), %rax + mov %rax, expected_rsp(%rip) + mov %rsp, %rax + mov %rax, resume_rsp(%rip) + lea .Lafter_file_fault(%rip), %rax + mov %rax, fault_resume(%rip) + movl $0, fault_seen(%rip) + call *%r12 +.Lafter_file_fault: + cmpl $1, fault_seen(%rip) + jne .Lfile_fault_not_seen + xor %eax, %eax + ret + +.Lfile_mapping_result: + mov %r13d, %eax + ret +.Lfile_fault_not_seen: + mov $27, %eax + ret + +.Lclose_mapping_fd: + mov $__NR_close, %eax + mov %r14, %rdi + syscall +.Lfile_mapping_failed: + mov $26, %eax + ret +.size test_file_mapping, .-test_file_mapping + +.type segv_handler, @function +segv_handler: + cmp $SIGSEGV, %edi + jne wrong_signal + mov expected_rsp(%rip), %rax + cmp %rax, UCONTEXT_RSP_OFFSET(%rdx) + jne wrong_fault_rsp + mov resume_rsp(%rip), %rax + mov %rax, UCONTEXT_RSP_OFFSET(%rdx) + mov fault_resume(%rip), %rax + mov %rax, UCONTEXT_RIP_OFFSET(%rdx) + movl $1, fault_seen(%rip) + ret +.size segv_handler, .-segv_handler + +wrong_signal: + mov $30, %edi + jmp do_exit +wrong_fault_rsp: + mov $31, %edi + jmp do_exit + +exit: + mov %eax, %edi +do_exit: + mov $__NR_exit, %eax + syscall + +.type signal_restorer, @function +signal_restorer: + mov $__NR_rt_sigreturn, %eax + syscall +.size signal_restorer, .-signal_restorer + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/registrations/process/meson.build b/tests/integration/registrations/process/meson.build index 0513ccc7c13..ac2f2afc2f9 100644 --- a/tests/integration/registrations/process/meson.build +++ b/tests/integration/registrations/process/meson.build @@ -39,6 +39,15 @@ if 'x86_64-linux-user' in target_dirs files('../../guest-fatal-signal-status.S'), ], }] + latx_integration_tests += [{ + 'name': 'test-push-pop-sp', + 'runner': find_program('../../test-push-pop-sp.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../push-pop-sp.S'), + files('../../push-pop-sp-file.S'), + ], + }] if 'CONFIG_LATX_KZT' in config_host latx_integration_tests += [{ 'name': 'test-int3-sigtrap-rip', @@ -52,6 +61,14 @@ if 'x86_64-linux-user' in target_dirs endif if 'i386-linux-user' in target_dirs + latx_integration_tests += [{ + 'name': 'test-push-pop-sp-i386', + 'runner': find_program('../../test-push-pop-sp-i386.sh'), + 'args': [ + emulators['latx-i386'], + files('../../push-pop-sp-i386.S'), + ], + }] latx_integration_tests += [{ 'name': 'test-fd-transform-race-i386', 'runner': find_program('../../test-fd-transform-race-i386.sh'), diff --git a/tests/integration/test-push-pop-sp-i386.sh b/tests/integration/test-push-pop-sp-i386.sh new file mode 100755 index 00000000000..940e2f30ad7 --- /dev/null +++ b/tests/integration/test-push-pop-sp-i386.sh @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu + +emulator=$(readlink -f "$1") +source_file=$(readlink -f "$2") +tmp_root=${TMPDIR:-${HOME}/tmp} +mkdir -p "$tmp_root" +workdir=$(mktemp -d "$tmp_root/push-pop-sp-i386.XXXXXX") +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the i386 guest" + exit 77 +fi + +"$clang" --target=i386-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none "$source_file" -o "$workdir/push-pop-sp-i386" + +set +e +timeout 10s env LATX_AOT=0 LATX_KZT=0 \ + "$emulator" "$workdir/push-pop-sp-i386" +ret=$? +set -e + +case $ret in +0) ;; +10) echo "FAIL: i386 rt_sigaction failed" >&2; exit "$ret" ;; +20) echo "FAIL: 32-bit PUSH/POP values or ESP mismatch" >&2; exit "$ret" ;; +21) echo "FAIL: 32-bit PUSH ESP semantics mismatch" >&2; exit "$ret" ;; +22) + echo "FAIL: PUSHAL/POPAL, PUSHFL/POPFL, or ENTER/LEAVE mismatch" >&2 + exit "$ret" + ;; +23) echo "FAIL: 32-bit helper boundary did not preserve ESP" >&2; exit "$ret" ;; +24) echo "FAIL: 32-bit fault handler was not called" >&2; exit "$ret" ;; +25) echo "FAIL: 32-bit ESP was not restored after the fault" >&2; exit "$ret" ;; +30) echo "FAIL: wrong signal reached the i386 handler" >&2; exit "$ret" ;; +31) echo "FAIL: fault ucontext contained the wrong ESP" >&2; exit "$ret" ;; +124) echo "FAIL: 32-bit PUSH/POP test timed out" >&2; exit "$ret" ;; +*) echo "FAIL: unexpected exit status $ret" >&2; exit "$ret" ;; +esac + +echo "PASS: 32-bit PUSH/POP, stack instructions, and fault ESP restore" diff --git a/tests/integration/test-push-pop-sp.sh b/tests/integration/test-push-pop-sp.sh new file mode 100755 index 00000000000..da279404dee --- /dev/null +++ b/tests/integration/test-push-pop-sp.sh @@ -0,0 +1,68 @@ +#!/bin/sh +set -eu + +emulator=$(readlink -f "$1") +source_file=$(readlink -f "$2") +file_source=$(readlink -f "$3") +tmp_root=${TMPDIR:-${HOME}/tmp} +mkdir -p "$tmp_root" +workdir=$(mktemp -d "$tmp_root/push-pop-sp.XXXXXX") +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +if command -v llvm-objcopy-19 >/dev/null 2>&1; then + objcopy=llvm-objcopy-19 +elif command -v llvm-objcopy >/dev/null 2>&1; then + objcopy=llvm-objcopy +else + echo "SKIP: llvm-objcopy is required to build the file-mapped guest code" + exit 77 +fi + +"$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none "$source_file" -o "$workdir/push-pop-sp" +"$clang" --target=x86_64-linux-gnu -c "$file_source" \ + -o "$workdir/push-pop-sp-file.o" +"$objcopy" -O binary --only-section=.text "$workdir/push-pop-sp-file.o" \ + "$workdir/push-pop-sp-file.bin" + +set +e +timeout 10s env LATX_AOT=0 LATX_KZT=0 \ + "$emulator" "$workdir/push-pop-sp" "$workdir/push-pop-sp-file.bin" +ret=$? +set -e + +case $ret in +0) ;; +10) echo "FAIL: rt_sigaction failed" >&2; exit "$ret" ;; +11) echo "FAIL: sigaltstack setup failed" >&2; exit "$ret" ;; +20) echo "FAIL: consecutive PUSH/POP values or RSP mismatch" >&2; exit "$ret" ;; +21) echo "FAIL: RSP memory or alias use mismatch" >&2; exit "$ret" ;; +22) echo "FAIL: PUSHF/POPF or ENTER/LEAVE mismatch" >&2; exit "$ret" ;; +23) echo "FAIL: helper boundary did not preserve RSP" >&2; exit "$ret" ;; +24) echo "FAIL: fault handler was not called" >&2; exit "$ret" ;; +25) echo "FAIL: RSP was not restored after the fault" >&2; exit "$ret" ;; +26) echo "FAIL: file-backed code mapping failed" >&2; exit "$ret" ;; +27) echo "FAIL: file-backed fault handler was not called" >&2; exit "$ret" ;; +28) + echo "FAIL: guard-page PUSH fault handler was not called" >&2 + exit "$ret" + ;; +29) echo "FAIL: guard-page PUSH setup failed" >&2; exit "$ret" ;; +30) echo "FAIL: wrong signal reached the handler" >&2; exit "$ret" ;; +31) echo "FAIL: fault ucontext contained the wrong RSP" >&2; exit "$ret" ;; +40) echo "FAIL: file-backed PUSH/POP values or RSP mismatch" >&2; exit "$ret" ;; +124) echo "FAIL: PUSH/POP RSP test timed out" >&2; exit "$ret" ;; +*) echo "FAIL: unexpected exit status $ret" >&2; exit "$ret" ;; +esac + +echo "PASS: PUSH/POP widths, RSP uses, helper, fault, guard page," \ + "and file mapping" From 1853206f1611346587c39afae4a49f93b8f0a547 Mon Sep 17 00:00:00 2001 From: yuerengan Date: Wed, 2 Sep 2026 17:18:52 +0800 Subject: [PATCH 2/3] LATX: share helper save and restore code Generate process-wide default and no-FP prologue and epilogue stubs for helpers that use common save and restore contracts. Prepare arguments and call the original helper between the stubs so argument registers do not require helper-specific handling. Route full-state instruction helpers and no-FP SMC store helpers through the matching stubs. Preserve address temporaries that remain live across a shared prologue because its independent register allocator may reuse the same host register. Keep FP80 conversion, return-register-sensitive paths, and every original inline SMC save and restore path as fallbacks when LATX_STATIC_HELPER=0. Relocate all four stub addresses for AOT and retain strict runtime and statistics switches. Add integration coverage for live RIP-relative helper arguments, common and no-FP helper paths, asynchronous signals, self-modifying stores, guest state, option validation, and inline/static equivalence. Signed-off-by: yuerengan --- linux-user/main.c | 31 ++ target/i386/latx/include/aot.h | 4 + target/i386/latx/include/latx-options.h | 4 + target/i386/latx/include/translate.h | 6 + target/i386/latx/latx-options.c | 4 + target/i386/latx/sbt/aot.c | 18 +- target/i386/latx/translator/tr-mov.c | 16 +- target/i386/latx/translator/tr-simd-mov.c | 34 +- target/i386/latx/translator/translate.c | 349 +++++++++++++++--- .../registrations/process/meson.build | 16 + tests/integration/static-helper-live-temp.S | 41 ++ tests/integration/static-helper-stubs.S | 287 ++++++++++++++ .../test-static-helper-live-temp.sh | 38 ++ tests/integration/test-static-helper-stubs.sh | 71 ++++ 14 files changed, 854 insertions(+), 65 deletions(-) create mode 100644 tests/integration/static-helper-live-temp.S create mode 100644 tests/integration/static-helper-stubs.S create mode 100755 tests/integration/test-static-helper-live-temp.sh create mode 100755 tests/integration/test-static-helper-stubs.sh diff --git a/linux-user/main.c b/linux-user/main.c index f8168cfdec2..9c8b17ddd70 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -635,6 +635,31 @@ static void handle_arg_latx_vpaes(const char *arg) option_vpaes = strtol(arg, NULL, 0); } +static void handle_arg_latx_static_helper(const char *arg) +{ + int value; + + if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) { + error_report("LATX_STATIC_HELPER must be exactly 0 or 1 (got '%s')", + arg); + exit(EXIT_FAILURE); + } + option_static_helper = value; +} + +static void handle_arg_latx_static_helper_stats(const char *arg) +{ + int value; + + if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) { + error_report( + "LATX_STATIC_HELPER_STATS must be exactly 0 or 1 (got '%s')", + arg); + exit(EXIT_FAILURE); + } + option_static_helper_stats = value; +} + static void handle_arg_latx_parallel(const char *arg) { close_latx_parallel = strtol(arg, NULL, 0); @@ -906,6 +931,12 @@ static const struct qemu_argument arg_table[] = { "", "specify enabled optimize type"}, {"latx-vpaes", "LATX_VPAES", true, handle_arg_latx_vpaes, "", "enable vpaes AES translation"}, + {"latx-static-helper", "LATX_STATIC_HELPER", true, + handle_arg_latx_static_helper, "0|1", + "use a shared save/call/restore stub for selected helpers"}, + {"latx-static-helper-stats", "LATX_STATIC_HELPER_STATS", true, + handle_arg_latx_static_helper_stats, "0|1", + "report translated callsites for static helper stubs"}, {"latx-smc", "LATX_SMC", true, handle_arg_latx_smc, "", "smc strategy: 0 (page) 1 (tb) 2(+shmm) 6(+helper,default)"}, {"latx-close-parallel", "LATX_CLOSE_PARALLEL", true, handle_arg_latx_parallel, diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index f72613a814d..43dc6fce9b0 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -260,6 +260,10 @@ typedef enum aot_rel_kind { LOAD_HELPER_CVTPH2PS_XMM, LOAD_HELPER_CVTPS2PH_YMM, LOAD_HELPER_CVTPS2PH_XMM, + LOAD_STATIC_HELPER_PROLOGUE, + LOAD_STATIC_HELPER_EPILOGUE, + LOAD_STATIC_HELPER_NOFP_PROLOGUE, + LOAD_STATIC_HELPER_NOFP_EPILOGUE, LOAD_HELPER_END, diff --git a/target/i386/latx/include/latx-options.h b/target/i386/latx/include/latx-options.h index e4d86cf46a1..50a8f4bcd2f 100644 --- a/target/i386/latx/include/latx-options.h +++ b/target/i386/latx/include/latx-options.h @@ -89,6 +89,8 @@ extern int option_fputag; extern int option_save_xmm; extern int option_enable_lasx; extern int option_vpaes; +extern int option_static_helper; +extern int option_static_helper_stats; extern int option_split_tb; extern int option_anonym; extern int option_imm_reg; @@ -125,6 +127,8 @@ extern unsigned long long counter_mips_tr; #define ENVSUP_LATX \ ENVFUN(LATX_OPTIMIZE, handle_arg_optimize) \ ENVFUN(LATX_VPAES, handle_arg_latx_vpaes) \ + ENVFUN(LATX_STATIC_HELPER, handle_arg_latx_static_helper) \ + ENVFUN(LATX_STATIC_HELPER_STATS, handle_arg_latx_static_helper_stats) \ ENVFUN(LATX_SMC, handle_arg_latx_smc) \ ENVFUN(LATX_CLOSE_PARALLEL, handle_arg_latx_parallel) \ ENVFUN(LATX_SOFTFPU, handle_arg_latx_softfpu) \ diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 3a19a698933..84315a63c6b 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1710,6 +1710,10 @@ void tr_generate_goto_tb(void); /* TODO */ extern ADDR native_rotate_fpu_by; extern ADDR indirect_jmp_glue; extern ADDR parallel_indirect_jmp_glue; +extern ADDR static_helper_prologue; +extern ADDR static_helper_epilogue; +extern ADDR static_helper_nofp_prologue; +extern ADDR static_helper_nofp_epilogue; void rotate_fpu_to_top(int top); void rotate_fpu_by(int step); void rotate_fpu_to_bias(int bias); @@ -1724,6 +1728,8 @@ void tr_gen_call_to_helper_cvttpd2pi(ADDR, int, int, enum aot_rel_kind); void tr_gen_call_to_helper_pclmulqdq(ADDR, int, int, int, int ,int, enum aot_rel_kind ); void tr_gen_call_to_helper_aes(ADDR, int, int, int, enum aot_rel_kind); +bool tr_gen_call_to_static_helper_nofp_prologue(IR2_OPND live_opnd); +void tr_gen_call_to_static_helper_nofp_epilogue(void); void tr_load_top_from_env(void); void tr_gen_top_mode_init(void); diff --git a/target/i386/latx/latx-options.c b/target/i386/latx/latx-options.c index 43671bbb556..21ed7e37643 100644 --- a/target/i386/latx/latx-options.c +++ b/target/i386/latx/latx-options.c @@ -97,6 +97,8 @@ int option_fputag; int option_save_xmm; int option_enable_lasx; int option_vpaes; +int option_static_helper; +int option_static_helper_stats; int option_split_tb; int option_anonym; int option_mem_test; @@ -236,6 +238,8 @@ void options_init(void) option_latx_disassemble_trace_cmp = 0; option_enable_lasx = 1; option_vpaes = 0; + option_static_helper = 1; + option_static_helper_stats = 0; counter_tb_exec = 0; counter_tb_tr = 0; diff --git a/target/i386/latx/sbt/aot.c b/target/i386/latx/sbt/aot.c index 3aa6292cfac..12d55f8a99e 100644 --- a/target/i386/latx/sbt/aot.c +++ b/target/i386/latx/sbt/aot.c @@ -1298,8 +1298,6 @@ static void* relkind_to_fixup_addr[] = { [LOAD_HELPER_CVTPS2PH_YMM] = helper_cvtps2ph_ymm, [LOAD_HELPER_CVTPS2PH_XMM] = helper_cvtps2ph_xmm, #endif - - }; void aot_do_tb_reloc(TranslationBlock *tb, struct aot_tb *stb, @@ -1418,7 +1416,21 @@ void aot_do_tb_reloc(TranslationBlock *tb, struct aot_tb *stb, } break; case LOAD_HELPER_BEGIN ... LOAD_HELPER_END: - helper_address = (uintptr_t)relkind_to_fixup_addr[aot_rel_table[i].kind]; + if (aot_rel_table[i].kind == LOAD_STATIC_HELPER_PROLOGUE) { + helper_address = static_helper_prologue; + } else if (aot_rel_table[i].kind == + LOAD_STATIC_HELPER_EPILOGUE) { + helper_address = static_helper_epilogue; + } else if (aot_rel_table[i].kind == + LOAD_STATIC_HELPER_NOFP_PROLOGUE) { + helper_address = static_helper_nofp_prologue; + } else if (aot_rel_table[i].kind == + LOAD_STATIC_HELPER_NOFP_EPILOGUE) { + helper_address = static_helper_nofp_epilogue; + } else { + helper_address = (uintptr_t) + relkind_to_fixup_addr[aot_rel_table[i].kind]; + } lsassert(helper_address); lsassert((*pinsn & 0xfe000000) == 0x14000000); /* lu12i.w */ *pinsn &= 0xfe00001f; diff --git a/target/i386/latx/translator/tr-mov.c b/target/i386/latx/translator/tr-mov.c index dab5886138a..d7c9dddee38 100644 --- a/target/i386/latx/translator/tr-mov.c +++ b/target/i386/latx/translator/tr-mov.c @@ -404,10 +404,14 @@ static void translate_mov_from_gpr(IR1_OPND *opnd0, IR1_OPND *opnd1) li_w(tmp, offset); la_add_d(tmp, mem, tmp); // save context - tr_save_registers_to_env(0xff, 0x0, 0x0, 0); + bool use_static_helper = + tr_gen_call_to_static_helper_nofp_prologue(tmp); + if (!use_static_helper) { + tr_save_registers_to_env(0xff, 0x0, 0x0, 0); #ifdef TARGET_X86_64 - tr_save_x64_8_registers_to_env(0xff, 0x0); + tr_save_x64_8_registers_to_env(0xff, 0x0); #endif + } // call smc_store_helper la_mov64(a0_ir2_opnd, env_ir2_opnd); la_mov64(a1_ir2_opnd, tmp); @@ -417,10 +421,14 @@ static void translate_mov_from_gpr(IR1_OPND *opnd0, IR1_OPND *opnd1) LOAD_HELPER_SMC_ST, 0); la_jirl(ra_ir2_opnd, tmp, 0); // restore context - tr_load_registers_from_env(0xff, 0x0, 0x0, 0); + if (use_static_helper) { + tr_gen_call_to_static_helper_nofp_epilogue(); + } else { + tr_load_registers_from_env(0xff, 0x0, 0x0, 0); #ifdef TARGET_X86_64 - tr_load_x64_8_registers_from_env(0xff, 0x0); + tr_load_x64_8_registers_from_env(0xff, 0x0); #endif + } // beq a0, finish la_beq(a0_ir2_opnd, zero_ir2_opnd, label_finish); store_ireg_to_ir1(src, opnd0, false); diff --git a/target/i386/latx/translator/tr-simd-mov.c b/target/i386/latx/translator/tr-simd-mov.c index 062484abce1..bcbe07c3b75 100644 --- a/target/i386/latx/translator/tr-simd-mov.c +++ b/target/i386/latx/translator/tr-simd-mov.c @@ -348,10 +348,16 @@ bool translate_movaps_vst_x4(IR1_INST *pir1) } // save context - tr_save_registers_to_env(0xff, 0x0, xmm & 0xff, 0); + bool use_static_helper = + tr_gen_call_to_static_helper_nofp_prologue(tmp); + if (use_static_helper) { + tr_save_ymm_to_env(xmm); + } else { + tr_save_registers_to_env(0xff, 0x0, xmm & 0xff, 0); #ifdef TARGET_X86_64 - tr_save_x64_8_registers_to_env(0xff, (xmm >> 8) & 0xff); + tr_save_x64_8_registers_to_env(0xff, (xmm >> 8) & 0xff); #endif + } // call helper la_mov64(a0_ir2_opnd, env_ir2_opnd); @@ -372,10 +378,14 @@ bool translate_movaps_vst_x4(IR1_INST *pir1) la_jirl(ra_ir2_opnd, tmp, 0); // restore context - tr_load_registers_from_env(0xff, 0x0, 0x0, 0); + if (use_static_helper) { + tr_gen_call_to_static_helper_nofp_epilogue(); + } else { + tr_load_registers_from_env(0xff, 0x0, 0x0, 0); #ifdef TARGET_X86_64 - tr_load_x64_8_registers_from_env(0xff, 0x0); + tr_load_x64_8_registers_from_env(0xff, 0x0); #endif + } // beq a0, finish la_beq(a0_ir2_opnd, zero_ir2_opnd, label_finish); @@ -429,10 +439,14 @@ bool translate_movaps(IR1_INST *pir1) li_w(tmp, offset); la_add_d(tmp, mem, tmp); // save context - tr_save_registers_to_env(0xff, 0x0, 0x0, 0); + bool use_static_helper = + tr_gen_call_to_static_helper_nofp_prologue(tmp); + if (!use_static_helper) { + tr_save_registers_to_env(0xff, 0x0, 0x0, 0); #ifdef TARGET_X86_64 - tr_save_x64_8_registers_to_env(0xff, 0x0); + tr_save_x64_8_registers_to_env(0xff, 0x0); #endif + } // call smc_store_helper la_mov64(a0_ir2_opnd, env_ir2_opnd); la_mov64(a1_ir2_opnd, tmp); @@ -444,10 +458,14 @@ bool translate_movaps(IR1_INST *pir1) LOAD_HELPER_SMC_VST, 0); la_jirl(ra_ir2_opnd, tmp, 0); // restore context - tr_load_registers_from_env(0xff, 0x0, 0x0, 0); + if (use_static_helper) { + tr_gen_call_to_static_helper_nofp_epilogue(); + } else { + tr_load_registers_from_env(0xff, 0x0, 0x0, 0); #ifdef TARGET_X86_64 - tr_load_x64_8_registers_from_env(0xff, 0x0); + tr_load_x64_8_registers_from_env(0xff, 0x0); #endif + } // beq a0, finish la_beq(a0_ir2_opnd, zero_ir2_opnd, label_finish); store_freg128_to_ir1_mem(ra_alloc_xmm(ir1_opnd_base_reg_num(src)), diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index f5512184f97..6419bb53ca8 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -83,6 +83,22 @@ void *interpret_glue; ADDR native_rotate_fpu_by; /* native_rotate_fpu_by(step, return_address) */ ADDR indirect_jmp_glue; ADDR parallel_indirect_jmp_glue; +ADDR static_helper_prologue; +ADDR static_helper_epilogue; +ADDR static_helper_nofp_prologue; +ADDR static_helper_nofp_epilogue; +static uint64_t static_helper_stub_bytes; + +static void static_helper_stats_record(const char *mode, int generated_ir2) +{ + if (!option_static_helper_stats) { + return; + } + fprintf(stderr, + "[LATX][static-helper] translated-callsite mode=%s " + "generated-ir2=%d shared-stub-bytes=%" PRIu64 "\n", + mode, generated_ir2, static_helper_stub_bytes); +} #ifndef TARGET_X86_64 int GPR_USEDEF_TO_SAVE = 0x7; @@ -3095,6 +3111,115 @@ static int ss_generate_match_fail_native_code(void* code_buf){ #endif } +static int generate_static_helper_prologue(void *code_buf) +{ + TRANSLATION_DATA *lat_ctx = lsenv->tr_data; + int code_nr; + + tr_init(NULL); + tr_save_registers_to_env(0xff, 0xff, 0xff, + options_to_save()); +#ifdef TARGET_X86_64 + tr_save_x64_8_registers_to_env(0xff, 0xff); +#endif + la_jirl(zero_ir2_opnd, ra_ir2_opnd, 0); + + label_dispose(NULL, lat_ctx); + code_nr = tr_ir2_assemble(code_buf, lat_ctx->first_ir2); + tr_fini(false); + return code_nr; +} + +static int generate_static_helper_epilogue(void *code_buf) +{ + TRANSLATION_DATA *lat_ctx = lsenv->tr_data; + int code_nr; + + tr_init(NULL); +#ifdef TARGET_X86_64 + tr_load_x64_8_registers_from_env(0xff, 0xff); +#endif + tr_load_registers_from_env(0xff, 0xff, 0xff, options_to_save()); + la_jirl(zero_ir2_opnd, ra_ir2_opnd, 0); + + label_dispose(NULL, lat_ctx); + code_nr = tr_ir2_assemble(code_buf, lat_ctx->first_ir2); + tr_fini(false); + return code_nr; +} + +static int generate_static_helper_nofp_prologue(void *code_buf) +{ + TRANSLATION_DATA *lat_ctx = lsenv->tr_data; + int code_nr; + + tr_init(NULL); + tr_save_registers_to_env(0xff, 0, 0, 0); +#ifdef TARGET_X86_64 + tr_save_x64_8_registers_to_env(0xff, 0); +#endif + la_jirl(zero_ir2_opnd, ra_ir2_opnd, 0); + + label_dispose(NULL, lat_ctx); + code_nr = tr_ir2_assemble(code_buf, lat_ctx->first_ir2); + tr_fini(false); + return code_nr; +} + +static int generate_static_helper_nofp_epilogue(void *code_buf) +{ + TRANSLATION_DATA *lat_ctx = lsenv->tr_data; + int code_nr; + + tr_init(NULL); + tr_load_registers_from_env(0xff, 0, 0, 0); +#ifdef TARGET_X86_64 + tr_load_x64_8_registers_from_env(0xff, 0); +#endif + la_jirl(zero_ir2_opnd, ra_ir2_opnd, 0); + + label_dispose(NULL, lat_ctx); + code_nr = tr_ir2_assemble(code_buf, lat_ctx->first_ir2); + tr_fini(false); + return code_nr; +} + +static int generate_static_helper_stub_code(void *code_buf) +{ + int code_nr; + int total_code_nr = 0; + + static_helper_prologue = (ADDR)code_buf; + code_nr = generate_static_helper_prologue(code_buf); + total_code_nr += code_nr; + code_buf += code_nr * 4; + + static_helper_epilogue = (ADDR)code_buf; + code_nr = generate_static_helper_epilogue(code_buf); + total_code_nr += code_nr; + code_buf += code_nr * 4; + + static_helper_nofp_prologue = (ADDR)code_buf; + code_nr = generate_static_helper_nofp_prologue(code_buf); + total_code_nr += code_nr; + code_buf += code_nr * 4; + + static_helper_nofp_epilogue = (ADDR)code_buf; + code_nr = generate_static_helper_nofp_epilogue(code_buf); + total_code_nr += code_nr; + static_helper_stub_bytes = total_code_nr * 4; + + if (option_dump) { + qemu_log("[static helper] default prologue at %p, epilogue at %p; " + "nofp prologue at %p, epilogue at %p; size = %d\n", + (void *)static_helper_prologue, + (void *)static_helper_epilogue, + (void *)static_helper_nofp_prologue, + (void *)static_helper_nofp_epilogue, total_code_nr * 4); + } + return total_code_nr; +} + /* note: native_rotate_fpu_by rotate data between mapped fp registers instead * of the in memory env->fpregs */ @@ -3221,6 +3346,9 @@ int generate_native_rotate_fpu_by(void *code_buf_addr) total_insts_num += insts_num; code_buf += insts_num * 4; + insts_num = generate_static_helper_stub_code(code_buf); + total_insts_num += insts_num; + return total_insts_num; } @@ -4000,8 +4128,59 @@ void convert_fpregs_x80_to_64(void) } } -static void tr_gen_call_to_helper_prologue(int use_fp) +static void tr_gen_call_to_static_helper_stub(ADDR stub, + enum aot_rel_kind rel_kind) +{ + TranslationBlock *tb __attribute__((unused)) = NULL; + + if (option_aot) { + tb = (TranslationBlock *)lsenv->tr_data->curr_tb; + } +#ifdef CONFIG_LATX_IMM_REG + if (option_imm_reg) { + free_imm_reg_all(); + } +#endif + IR2_OPND stub_addr = ra_alloc_dbt_arg2(); + aot_load_host_addr(stub_addr, stub, rel_kind, 0); + la_jirl(ra_ir2_opnd, stub_addr, 0); +} + +bool tr_gen_call_to_static_helper_nofp_prologue(IR2_OPND live_opnd) +{ + int stats_ir2_start; + + if (!option_static_helper) { + return false; + } + + stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + /* The shared stub may reuse a caller itemp in its independent RA state. */ + helper_save_reg(live_opnd); + tr_gen_call_to_static_helper_stub(static_helper_nofp_prologue, + LOAD_STATIC_HELPER_NOFP_PROLOGUE); + helper_restore_reg(live_opnd); + static_helper_stats_record( + "static-nofp", + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); + return true; +} + +void tr_gen_call_to_static_helper_nofp_epilogue(void) { + tr_gen_call_to_static_helper_stub(static_helper_nofp_epilogue, + LOAD_STATIC_HELPER_NOFP_EPILOGUE); +} + +static bool tr_gen_call_to_helper_prologue(int use_fp, + bool static_compatible) +{ + if (option_static_helper && static_compatible && !use_fp) { + tr_gen_call_to_static_helper_stub(static_helper_prologue, + LOAD_STATIC_HELPER_PROLOGUE); + return true; + } + tr_save_registers_to_env(0, FPR_USEDEF_TO_SAVE, XMM_USEDEF_TO_SAVE, options_to_save()); #ifdef TARGET_X86_64 @@ -4022,10 +4201,17 @@ static void tr_gen_call_to_helper_prologue(int use_fp) la_mov64(a0_ir2_opnd, env_ir2_opnd); la_jirl(ra_ir2_opnd, func_addr_opnd, 0); } + return false; } -static void tr_gen_call_to_helper_epilogue(int use_fp) +static void tr_gen_call_to_helper_epilogue(int use_fp, bool use_static) { + if (use_static) { + tr_gen_call_to_static_helper_stub(static_helper_epilogue, + LOAD_STATIC_HELPER_EPILOGUE); + return; + } + if (use_fp) { IR2_OPND func_addr_opnd = ra_alloc_dbt_arg2(); TranslationBlock *tb __attribute__((unused)) = NULL; @@ -4044,9 +4230,61 @@ static void tr_gen_call_to_helper_epilogue(int use_fp) options_to_save()); } +static bool tr_gen_call_to_helper_full_prologue(void) +{ + if (option_static_helper) { + tr_gen_call_to_static_helper_stub(static_helper_prologue, + LOAD_STATIC_HELPER_PROLOGUE); + return true; + } + + tr_save_registers_to_env(0xff, FPR_USEDEF_TO_SAVE, 0xff, + options_to_save()); +#ifdef TARGET_X86_64 + tr_save_x64_8_registers_to_env(0xff, 0xff); +#endif + return false; +} + +static void tr_gen_call_to_helper_full_epilogue(bool use_static) +{ + if (use_static) { + tr_gen_call_to_static_helper_stub(static_helper_epilogue, + LOAD_STATIC_HELPER_EPILOGUE); + return; + } + + tr_load_registers_from_env(0xff, FPR_USEDEF_TO_SAVE, 0xff, + options_to_save()); +#ifdef TARGET_X86_64 + tr_load_x64_8_registers_from_env(0xff, 0xff); +#endif +} + +static const char *static_helper_mode(bool use_static, int use_fp, + bool static_compatible) +{ + if (use_static) { + return "static"; + } + if (!option_static_helper) { + return "inline-disabled"; + } + if (use_fp) { + return "inline-fp80"; + } + if (!static_compatible) { + return "inline-special"; + } + return "inline"; +} + /* helper with 1 default arg(CPUArchState*) */ void tr_gen_call_to_helper1(ADDR func, int use_fp, enum aot_rel_kind REL_KIND) { + int stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + bool use_static; + /* aot relocation requires the tb struct */ TranslationBlock *tb __attribute__((unused)) = NULL; if (option_aot) { @@ -4054,7 +4292,7 @@ void tr_gen_call_to_helper1(ADDR func, int use_fp, enum aot_rel_kind REL_KIND) } /* prologue */ - tr_gen_call_to_helper_prologue(use_fp); + use_static = tr_gen_call_to_helper_prologue(use_fp, true); /* load the helper addr */ IR2_OPND func_addr_opnd = ra_alloc_dbt_arg2(); @@ -4063,12 +4301,18 @@ void tr_gen_call_to_helper1(ADDR func, int use_fp, enum aot_rel_kind REL_KIND) /* jmp and epilogue */ la_mov64(a0_ir2_opnd, env_ir2_opnd); la_jirl(ra_ir2_opnd, func_addr_opnd, 0); - tr_gen_call_to_helper_epilogue(use_fp); + tr_gen_call_to_helper_epilogue(use_fp, use_static); + static_helper_stats_record(static_helper_mode(use_static, use_fp, true), + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); } void tr_gen_call_to_helper2(ADDR func, IR2_OPND mem_opnd, int use_fp, enum aot_rel_kind REL_KIND) { + int stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + bool preserve_mem_opnd = option_static_helper && !use_fp; + bool use_static; + /* aot relocation requires the tb struct */ TranslationBlock *tb __attribute__((unused)) = NULL; if (option_aot) { @@ -4076,7 +4320,14 @@ void tr_gen_call_to_helper2(ADDR func, IR2_OPND mem_opnd, int use_fp, } /* prologue */ - tr_gen_call_to_helper_prologue(use_fp); + /* mem_opnd remains live, but the shared stub has independent RA state. */ + if (preserve_mem_opnd) { + helper_save_reg(mem_opnd); + } + use_static = tr_gen_call_to_helper_prologue(use_fp, true); + if (use_static) { + helper_restore_reg(mem_opnd); + } /* load the helper addr */ IR2_OPND func_addr_opnd = ra_alloc_dbt_arg2(); @@ -4086,13 +4337,18 @@ void tr_gen_call_to_helper2(ADDR func, IR2_OPND mem_opnd, int use_fp, la_mov64(a0_ir2_opnd, env_ir2_opnd); la_mov64(a1_ir2_opnd, mem_opnd); la_jirl(ra_ir2_opnd, func_addr_opnd, 0); - tr_gen_call_to_helper_epilogue(use_fp); + tr_gen_call_to_helper_epilogue(use_fp, use_static); + static_helper_stats_record(static_helper_mode(use_static, use_fp, true), + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); } #ifdef CONFIG_LATX_AVX_OPT void tr_gen_call_to_helper_xgetbv(void) { + int stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + bool use_static; + /* aot relocation requires the tb struct */ TranslationBlock *tb __attribute__((unused)) = NULL; if (option_aot) { @@ -4100,7 +4356,7 @@ void tr_gen_call_to_helper_xgetbv(void) } /* prologue */ - tr_gen_call_to_helper_prologue(0); + use_static = tr_gen_call_to_helper_prologue(0, false); /* load func_addr and jmp */ IR2_OPND func_addr_opnd = ra_alloc_dbt_arg2(); @@ -4121,7 +4377,9 @@ void tr_gen_call_to_helper_xgetbv(void) la_bstrpick_d(eax_opnd, a0_ir2_opnd, 31, 0); la_bstrpick_d(edx_opnd, a0_ir2_opnd, 63, 32); - tr_gen_call_to_helper_epilogue(0); + tr_gen_call_to_helper_epilogue(0, use_static); + static_helper_stats_record(static_helper_mode(use_static, 0, false), + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); } #endif @@ -4248,6 +4506,9 @@ void gen_test_page_flag(IR2_OPND mem_opnd, int mem_imm, uint32_t flag) void tr_gen_call_to_helper_vfll(ADDR func, IR2_OPND arg1, IR2_OPND arg2, int use_fp, enum aot_rel_kind REL_KIND) { + int stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + bool use_static; + /* aot relocation requires the tb struct */ TranslationBlock *tb __attribute__((unused)) = NULL; if (option_aot) { @@ -4256,7 +4517,7 @@ void tr_gen_call_to_helper_vfll(ADDR func, IR2_OPND arg1, IR2_OPND arg2, helper_save_reg(arg1); helper_save_reg(arg2); /* prologue */ - tr_gen_call_to_helper_prologue(use_fp); + use_static = tr_gen_call_to_helper_prologue(use_fp, true); helper_restore_reg(arg1); helper_restore_reg(arg2); la_mov64(a0_ir2_opnd, env_ir2_opnd); @@ -4268,17 +4529,16 @@ void tr_gen_call_to_helper_vfll(ADDR func, IR2_OPND arg1, IR2_OPND arg2, aot_load_host_addr(func_addr_opnd, (ADDR)func, REL_KIND, 0); la_jirl(ra_ir2_opnd, func_addr_opnd, 0); - tr_gen_call_to_helper_epilogue(use_fp); + tr_gen_call_to_helper_epilogue(use_fp, use_static); + static_helper_stats_record(static_helper_mode(use_static, use_fp, true), + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); } void tr_gen_call_to_helper_cvttpd2pi(ADDR func, int dest_xmm_num, int src_xmm_num, enum aot_rel_kind REL_KIND) { - /* prologue */ - tr_save_registers_to_env(0xff, FPR_USEDEF_TO_SAVE, 0xff, options_to_save()); -#ifdef TARGET_X86_64 - tr_save_x64_8_registers_to_env(0xff, 0xff); -#endif + int stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + bool use_static = tr_gen_call_to_helper_full_prologue(); /* set arguments */ la_mov64(a0_ir2_opnd, env_ir2_opnd); @@ -4302,22 +4562,16 @@ void tr_gen_call_to_helper_cvttpd2pi(ADDR func, int dest_xmm_num, int src_xmm_nu aot_load_host_addr(func_addr_opnd, (ADDR)func, REL_KIND, 0); la_jirl(ra_ir2_opnd, func_addr_opnd, 0); - /* prologue, jmp and epilogue */ - tr_load_registers_from_env(0xff, FPR_USEDEF_TO_SAVE, 0xff, options_to_save()); -#ifdef TARGET_X86_64 - tr_load_x64_8_registers_from_env(0xff, 0xff); -#endif - + tr_gen_call_to_helper_full_epilogue(use_static); + static_helper_stats_record(static_helper_mode(use_static, 0, true), + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); } -void tr_gen_call_to_helper_pcmpxstrx(ADDR func, int dest_xmm_num, int src_xmm_num, int ctrl, - enum aot_rel_kind REL_KIND) +void tr_gen_call_to_helper_pcmpxstrx(ADDR func, int dest_xmm_num, + int src_xmm_num, int ctrl, enum aot_rel_kind REL_KIND) { - /* prologue */ - tr_save_registers_to_env(0xff, FPR_USEDEF_TO_SAVE, 0xff, options_to_save()); -#ifdef TARGET_X86_64 - tr_save_x64_8_registers_to_env(0xff, 0xff); -#endif + int stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + bool use_static = tr_gen_call_to_helper_full_prologue(); /* set arguments */ la_mov64(a0_ir2_opnd, env_ir2_opnd); @@ -4337,21 +4591,18 @@ void tr_gen_call_to_helper_pcmpxstrx(ADDR func, int dest_xmm_num, int src_xmm_nu #endif li_d(a3_ir2_opnd, ctrl); - /* load func_addr and jmp */ - IR2_OPND func_addr_opnd = ra_alloc_dbt_arg2(); - aot_load_host_addr(func_addr_opnd, (ADDR)func, REL_KIND, 0); - la_jirl(ra_ir2_opnd, func_addr_opnd, 0); - - /* prologue, jmp and epilogue */ - tr_load_registers_from_env(0xff, FPR_USEDEF_TO_SAVE, 0xff, options_to_save()); -#ifdef TARGET_X86_64 - tr_load_x64_8_registers_from_env(0xff, 0xff); -#endif + tr_gen_call_to_helper(func, REL_KIND); + tr_gen_call_to_helper_full_epilogue(use_static); + static_helper_stats_record(static_helper_mode(use_static, 0, true), + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); } void tr_gen_call_to_helper_pclmulqdq(ADDR func, int d, int s1, int s2, int ctrl,int use_fp, enum aot_rel_kind REL_KIND) { + int stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + bool use_static; + /* aot relocation requires the tb struct */ TranslationBlock *tb __attribute__((unused)) = NULL; if (option_aot) { @@ -4359,7 +4610,7 @@ void tr_gen_call_to_helper_pclmulqdq(ADDR func, int d, int s1, int s2, int ctrl } /* prologue */ - tr_gen_call_to_helper_prologue(use_fp); + use_static = tr_gen_call_to_helper_prologue(use_fp, true); /* set arguments */ la_mov64(a0_ir2_opnd, env_ir2_opnd); @@ -4388,17 +4639,17 @@ void tr_gen_call_to_helper_pclmulqdq(ADDR func, int d, int s1, int s2, int ctrl aot_load_host_addr(func_addr_opnd, (ADDR)func, REL_KIND, 0); la_jirl(ra_ir2_opnd, func_addr_opnd, 0); - tr_gen_call_to_helper_epilogue(use_fp); + tr_gen_call_to_helper_epilogue(use_fp, use_static); + static_helper_stats_record(static_helper_mode(use_static, use_fp, true), + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); } void tr_gen_call_to_helper_aes(ADDR func, int dest_xmm_num, int src1_xmm_num, int src2_xmm_num, enum aot_rel_kind REL_KIND) { - /* prologue */ - tr_save_registers_to_env(0xff, FPR_USEDEF_TO_SAVE, 0xff, options_to_save()); -#ifdef TARGET_X86_64 - tr_save_x64_8_registers_to_env(0xff, 0xff); -#endif + int stats_ir2_start = lsenv->tr_data->ir2_inst_num_current; + bool use_static = tr_gen_call_to_helper_full_prologue(); + /* set arguments */ la_mov64(a0_ir2_opnd, env_ir2_opnd); #ifndef TARGET_X86_64 @@ -4423,11 +4674,9 @@ void tr_gen_call_to_helper_aes(ADDR func, int dest_xmm_num, int src1_xmm_num, in IR2_OPND func_addr_opnd = ra_alloc_dbt_arg2(); aot_load_host_addr(func_addr_opnd, (ADDR)func, REL_KIND, 0); la_jirl(ra_ir2_opnd, func_addr_opnd, 0); - /* prologue, jmp and epilogue */ - tr_load_registers_from_env(0xff, FPR_USEDEF_TO_SAVE, 0xff, options_to_save()); -#ifdef TARGET_X86_64 - tr_load_x64_8_registers_from_env(0xff, 0xff); -#endif + tr_gen_call_to_helper_full_epilogue(use_static); + static_helper_stats_record(static_helper_mode(use_static, 0, true), + lsenv->tr_data->ir2_inst_num_current - stats_ir2_start); } IR2_OPND tr_lat_spin_lock(IR2_OPND mem_addr, int imm) diff --git a/tests/integration/registrations/process/meson.build b/tests/integration/registrations/process/meson.build index ac2f2afc2f9..6efe700b8b4 100644 --- a/tests/integration/registrations/process/meson.build +++ b/tests/integration/registrations/process/meson.build @@ -48,6 +48,22 @@ if 'x86_64-linux-user' in target_dirs files('../../push-pop-sp-file.S'), ], }] + latx_integration_tests += [{ + 'name': 'test-static-helper-stubs', + 'runner': find_program('../../test-static-helper-stubs.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../static-helper-stubs.S'), + ], + }] + latx_integration_tests += [{ + 'name': 'test-static-helper-live-temp', + 'runner': find_program('../../test-static-helper-live-temp.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../static-helper-live-temp.S'), + ], + }] if 'CONFIG_LATX_KZT' in config_host latx_integration_tests += [{ 'name': 'test-int3-sigtrap-rip', diff --git a/tests/integration/static-helper-live-temp.S b/tests/integration/static-helper-live-temp.S new file mode 100644 index 00000000000..da8282862dc --- /dev/null +++ b/tests/integration/static-helper-live-temp.S @@ -0,0 +1,41 @@ +.equ __NR_exit, 60 + +.section .rodata +alternate_fcw: + .word 0x027f + +.section .bss +.align 64 +fx_state: + .skip 512 +restored_fcw: + .skip 2 + +.section .text +.global _start +.type _start, @function +_start: + fninit + fxsave64 fx_state(%rip) + cmpw $0x037f, fx_state(%rip) + jne .Lfxsave_failed + + fldcw alternate_fcw(%rip) + fxrstor64 fx_state(%rip) + fnstcw restored_fcw(%rip) + cmpw $0x037f, restored_fcw(%rip) + jne .Lfxrstor_failed + + xor %edi, %edi + jmp .Lexit +.Lfxsave_failed: + mov $42, %edi + jmp .Lexit +.Lfxrstor_failed: + mov $43, %edi +.Lexit: + mov $__NR_exit, %eax + syscall +.size _start, .-_start + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/static-helper-stubs.S b/tests/integration/static-helper-stubs.S new file mode 100644 index 00000000000..60b461e8571 --- /dev/null +++ b/tests/integration/static-helper-stubs.S @@ -0,0 +1,287 @@ +.equ __NR_write, 1 +.equ __NR_rt_sigaction, 13 +.equ __NR_rt_sigreturn, 15 +.equ __NR_mprotect, 10 +.equ __NR_setitimer, 38 +.equ __NR_exit, 60 +.equ SIGALRM, 14 +.equ SA_RESTORER, 0x04000000 +.equ RESULT_SIZE, 288 +.equ EFLAGS_MASK, 0x8c5 + +.section .rodata +.align 32 +aes_input: + .byte 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77 + .byte 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff +aes_key: + .byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08 + .byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 +ymm_value: + .quad 0x0123456789abcdef, 0xfedcba9876543210 + .quad 0x55aa55aa55aa55aa, 0xaa55aa55aa55aa55 +pcmp_left: + .ascii "static-helper-01" +pcmp_right: + .ascii "static-helper-02" +.align 8 +alarm_action: + .quad alarm_handler + .quad SA_RESTORER + .quad signal_restorer + .quad 0 +alarm_timer: + .quad 0, 1000, 0, 1000 +alarm_timer_off: + .quad 0, 0, 0, 0 + +.section .bss +.align 32 +result: + .skip RESULT_SIZE +.align 4 +alarm_count: + .long 0 + +.section .text +.global _start +.type _start, @function +_start: + mov $__NR_rt_sigaction, %eax + mov $SIGALRM, %edi + lea alarm_action(%rip), %rsi + xor %edx, %edx + mov $8, %r10d + syscall + test %rax, %rax + js .Lsignal_failed + + mov $__NR_setitimer, %eax + xor %edi, %edi + lea alarm_timer(%rip), %rsi + xor %edx, %edx + syscall + test %rax, %rax + js .Lsignal_failed + + pxor %xmm0, %xmm0 + pxor %xmm1, %xmm1 + mov $1000000, %r12d +.Lsignal_loop: + pcmpistri $0, %xmm0, %xmm1 + dec %r12d + jnz .Lsignal_loop + + mov $__NR_setitimer, %eax + xor %edi, %edi + lea alarm_timer_off(%rip), %rsi + xor %edx, %edx + syscall + cmpl $0, alarm_count(%rip) + je .Lsignal_failed + + lea smc_target(%rip), %rdi + and $-4096, %rdi + mov $4096, %esi + mov $7, %edx + mov $__NR_mprotect, %eax + syscall + test %rax, %rax + js .Lsmc_failed + + call smc_target + mov $300, %ecx +.Lsmc_st_loop: + mov %rcx, smc_data(%rip) + call smc_target + cmp $42, %eax + jne .Lsmc_failed + dec %ecx + jnz .Lsmc_st_loop + cmpq $1, smc_data(%rip) + jne .Lsmc_failed + + pxor %xmm0, %xmm0 + mov $300, %ecx +.Lsmc_vst_loop: + movaps %xmm0, smc_data(%rip) + call smc_target + cmp $42, %eax + jne .Lsmc_failed + dec %ecx + jnz .Lsmc_vst_loop + cmpq $0, smc_data(%rip) + jne .Lsmc_failed + cmpq $0, smc_data+8(%rip) + jne .Lsmc_failed + + pxor %xmm0, %xmm0 + pcmpeqb %xmm1, %xmm1 + movdqa %xmm1, %xmm2 + psllq $1, %xmm2 + movdqa %xmm2, %xmm3 + mov $300, %ecx +.Lsmc_vst_x4_loop: + movaps %xmm0, smc_data(%rip) + movaps %xmm1, smc_data+16(%rip) + movaps %xmm2, smc_data+32(%rip) + movaps %xmm3, smc_data+48(%rip) + call smc_target + cmp $42, %eax + jne .Lsmc_failed + dec %ecx + jnz .Lsmc_vst_x4_loop + cmpq $0, smc_data(%rip) + jne .Lsmc_failed + cmpq $0, smc_data+8(%rip) + jne .Lsmc_failed + cmpq $-1, smc_data+16(%rip) + jne .Lsmc_failed + cmpq $-1, smc_data+24(%rip) + jne .Lsmc_failed + cmpq $-2, smc_data+32(%rip) + jne .Lsmc_failed + cmpq $-2, smc_data+40(%rip) + jne .Lsmc_failed + cmpq $-2, smc_data+48(%rip) + jne .Lsmc_failed + cmpq $-2, smc_data+56(%rip) + jne .Lsmc_failed + + movabs $0x1020304050607080, %rbx + movabs $0x1122334455667788, %rbp + movabs $0x2030405060708090, %r8 + movabs $0x30405060708090a0, %r9 + movabs $0x405060708090a0b0, %r10 + movabs $0x5060708090a0b0c0, %r11 + movabs $0x60708090a0b0c0d0, %r12 + movabs $0x708090a0b0c0d0e0, %r13 + movabs $0x8090a0b0c0d0e0f0, %r14 + movabs $0x90a0b0c0d0e0f001, %r15 + + fninit + fld1 + fldpi + vmovdqu ymm_value(%rip), %ymm15 + movdqa aes_input(%rip), %xmm0 + movdqa aes_key(%rip), %xmm1 + movdqa pcmp_left(%rip), %xmm7 + + mov $0x7fffffff, %eax + add $1, %eax + stc + aesenc %xmm1, %xmm0 + pushfq + pop %rax + and $EFLAGS_MASK, %eax + mov %rax, result+64(%rip) + + movdqu %xmm0, result(%rip) + movdqu %xmm7, result+16(%rip) + vmovdqu %ymm15, result+32(%rip) + + movdqa aes_input(%rip), %xmm8 + movdqa aes_key(%rip), %xmm9 + pclmulqdq $0, %xmm9, %xmm8 + + movdqa pcmp_left(%rip), %xmm2 + movdqa pcmp_right(%rip), %xmm3 + pcmpistri $0x18, %xmm3, %xmm2 + mov %rcx, result+184(%rip) + pushfq + pop %rax + and $EFLAGS_MASK, %eax + mov %rax, result+192(%rip) + + movdqa pcmp_left(%rip), %xmm2 + movdqa pcmp_right(%rip), %xmm3 + pcmpistrm $0x18, %xmm3, %xmm2 + movdqu %xmm0, result+200(%rip) + + movdqa aes_key(%rip), %xmm4 + aeskeygenassist $0x1b, %xmm4, %xmm5 + movdqu %xmm5, result+216(%rip) + aesimc %xmm5, %xmm6 + movdqu %xmm6, result+232(%rip) + + mov $16, %eax + mov $16, %edx + movdqa pcmp_left(%rip), %xmm2 + movdqa pcmp_right(%rip), %xmm3 + pcmpestri $0x18, %xmm3, %xmm2 + mov %rcx, result+248(%rip) + pushfq + pop %rax + and $EFLAGS_MASK, %eax + mov %rax, result+256(%rip) + + mov $16, %eax + mov $16, %edx + movdqa pcmp_left(%rip), %xmm2 + movdqa pcmp_right(%rip), %xmm3 + pcmpestrm $0x18, %xmm3, %xmm2 + movdqu %xmm0, result+264(%rip) + + mov %rbx, result+72(%rip) + mov %rbp, result+80(%rip) + mov %r8, result+88(%rip) + mov %r9, result+96(%rip) + mov %r10, result+104(%rip) + mov %r11, result+112(%rip) + mov %r12, result+120(%rip) + mov %r13, result+128(%rip) + mov %r14, result+136(%rip) + mov %r15, result+144(%rip) + + fnstsw result+152(%rip) + fstpt result+160(%rip) + fstpt result+170(%rip) + vzeroupper + + mov $__NR_write, %eax + mov $1, %edi + lea result(%rip), %rsi + mov $RESULT_SIZE, %edx + syscall + cmp $RESULT_SIZE, %eax + jne .Lwrite_failed + + xor %edi, %edi + jmp .Lexit +.Lwrite_failed: + mov $1, %edi + jmp .Lexit +.Lsignal_failed: + mov $2, %edi + jmp .Lexit +.Lsmc_failed: + mov $3, %edi +.Lexit: + mov $__NR_exit, %eax + syscall +.size _start, .-_start + +.type alarm_handler, @function +alarm_handler: + incl alarm_count(%rip) + ret +.size alarm_handler, .-alarm_handler + +.type signal_restorer, @function +signal_restorer: + mov $__NR_rt_sigreturn, %eax + syscall +.size signal_restorer, .-signal_restorer + +.section .smc_text,"ax",@progbits +.balign 65536 +.type smc_target, @function +smc_target: + mov $42, %eax + ret +.size smc_target, .-smc_target +.balign 16 +smc_data: + .fill 64, 1, 0 + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/test-static-helper-live-temp.sh b/tests/integration/test-static-helper-live-temp.sh new file mode 100755 index 00000000000..747e1a1fef6 --- /dev/null +++ b/tests/integration/test-static-helper-live-temp.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +emulator=$(readlink -f "$1") +source_file=$(readlink -f "$2") +tmp_root=${TMPDIR:-${HOME}/tmp} +mkdir -p "$tmp_root" +workdir=$(mktemp -d "$tmp_root/static-helper-live-temp.XXXXXX") +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +"$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none "$source_file" -o "$workdir/live-temp" + +run_guest() +{ + mode=$1 + timeout 10s env LATX_AOT=0 LATX_KZT=0 LATX_SOFTFPU=0 \ + LATX_STATIC_HELPER="$mode" "$emulator" "$workdir/live-temp" +} + +run_guest 0 +run_guest 1 + +timeout 10s env LATX_AOT=0 LATX_KZT=0 LATX_SOFTFPU=0 \ + LATX_STATIC_HELPER=1 LATX_STATIC_HELPER_STATS=1 \ + "$emulator" "$workdir/live-temp" 2>"$workdir/stats.err" +grep -q 'translated-callsite mode=static generated-ir2=' "$workdir/stats.err" + +echo "PASS: static helper prologue preserves live RIP-relative addresses" diff --git a/tests/integration/test-static-helper-stubs.sh b/tests/integration/test-static-helper-stubs.sh new file mode 100755 index 00000000000..c32de5c6fc1 --- /dev/null +++ b/tests/integration/test-static-helper-stubs.sh @@ -0,0 +1,71 @@ +#!/bin/sh +set -eu + +emulator=$(readlink -f "$1") +source_file=$(readlink -f "$2") +tmp_root=${TMPDIR:-${HOME}/tmp} +mkdir -p "$tmp_root" +workdir=$(mktemp -d "$tmp_root/static-helper-stubs.XXXXXX") +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +"$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none "$source_file" -o "$workdir/static-helper-stubs" + +run_guest() +{ + mode=$1 + output=$2 + + timeout 10s env LATX_AOT=0 LATX_KZT=0 LATX_VPAES=0 LATX_SMC=6 \ + LATX_STATIC_HELPER="$mode" \ + "$emulator" "$workdir/static-helper-stubs" >"$output" + test "$(wc -c <"$output")" -eq 288 +} + +run_guest 0 "$workdir/inline.out" +run_guest 1 "$workdir/static.out" +cmp "$workdir/inline.out" "$workdir/static.out" + +timeout 10s env LATX_AOT=0 LATX_KZT=0 LATX_VPAES=0 LATX_SMC=6 \ + "$emulator" "$workdir/static-helper-stubs" >"$workdir/default.out" +cmp "$workdir/static.out" "$workdir/default.out" + +if env LATX_STATIC_HELPER=invalid "$emulator" \ + "$workdir/static-helper-stubs" >/dev/null 2>&1; then + echo "FAIL: invalid LATX_STATIC_HELPER was accepted" >&2 + exit 1 +fi +if env LATX_STATIC_HELPER_STATS=2 "$emulator" \ + "$workdir/static-helper-stubs" >/dev/null 2>&1; then + echo "FAIL: invalid LATX_STATIC_HELPER_STATS was accepted" >&2 + exit 1 +fi + +timeout 10s env LATX_AOT=0 LATX_KZT=0 LATX_VPAES=0 LATX_SMC=6 \ + LATX_STATIC_HELPER=1 LATX_STATIC_HELPER_STATS=1 \ + "$emulator" "$workdir/static-helper-stubs" >/dev/null \ + 2>"$workdir/stats.err" +grep -q 'translated-callsite mode=static generated-ir2=' "$workdir/stats.err" +test "$(grep -c 'translated-callsite mode=static ' "$workdir/stats.err")" \ + -ge 8 +test "$(grep -c 'translated-callsite mode=static-nofp ' \ + "$workdir/stats.err")" -ge 3 + +timeout 10s env LATX_AOT=0 LATX_KZT=0 LATX_VPAES=0 LATX_SMC=6 \ + LATX_STATIC_HELPER=0 LATX_STATIC_HELPER_STATS=1 \ + "$emulator" "$workdir/static-helper-stubs" >/dev/null \ + 2>"$workdir/inline-stats.err" +grep -q 'translated-callsite mode=inline-disabled generated-ir2=' \ + "$workdir/inline-stats.err" + +echo "PASS: generic static helper stubs preserve signal, SMC, GPR, EFLAGS," \ + "x87, XMM, and YMM state" From 442a29c4f7cb59a1dd1efb92a6574a5855d3a1cc Mon Sep 17 00:00:00 2001 From: yuerengan Date: Thu, 3 Sep 2026 14:42:32 +0800 Subject: [PATCH 3/3] LATX: invalidate AOT cache on static helper mode changes Record the LATX_STATIC_HELPER mode in every generated AOT header and reject cached code when the current runtime mode differs. This keeps the rollback switch effective for previously generated code and makes mode changes fall back to JIT regeneration. Bump the AOT footer version so caches written before the new header field are never interpreted using padding data. Apply the same compatibility check to normal and merged AOT loads, and cover both mode transitions in the cache reader test. Signed-off-by: yuerengan --- target/i386/latx/include/aot.h | 5 +++-- target/i386/latx/sbt/aot.c | 1 + target/i386/latx/sbt/aot_merge.c | 16 +++++++++++++++- target/i386/latx/sbt/aot_reader.c | 17 +++++++++++++++++ .../latx/sbt/tests/aot-cache-reader-test.c | 18 ++++++++++++++++++ 5 files changed, 54 insertions(+), 3 deletions(-) diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index 43dc6fce9b0..359d14d1068 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -39,9 +39,9 @@ extern const char *aot_left_file_minsize_optarg; * +--------------+ */ #ifdef CONFIG_LATX_DEBUG -#define AOT_VERSION "Version: "LATX_VERSION"-debug" +#define AOT_VERSION "Version: "LATX_VERSION"-debug-shcfg1" #else -#define AOT_VERSION "Version: "LATX_VERSION"-release" +#define AOT_VERSION "Version: "LATX_VERSION"-release-shcfg1" #endif typedef struct aot_header { uint32_t lib_size; @@ -57,6 +57,7 @@ typedef struct aot_header { #define CACHE_AOT_FILE 4 #define HASH_AOT_FILE 8 uint8_t aot_file_type; + uint8_t static_helper; } aot_header; typedef struct aot_file_info { diff --git a/target/i386/latx/sbt/aot.c b/target/i386/latx/sbt/aot.c index 12d55f8a99e..b44869325c1 100644 --- a/target/i386/latx/sbt/aot.c +++ b/target/i386/latx/sbt/aot.c @@ -766,6 +766,7 @@ int do_generate_aot(int first_seg_in_lib, int end_seg_in_lib) p_header->aot_file_type = seg_info_vector[first_seg_in_lib]->aot_file_type; + p_header->static_helper = !!option_static_helper; if (p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE)) { struct stat statbuf; diff --git a/target/i386/latx/sbt/aot_merge.c b/target/i386/latx/sbt/aot_merge.c index 16ce5cdf721..4bd87474bce 100644 --- a/target/i386/latx/sbt/aot_merge.c +++ b/target/i386/latx/sbt/aot_merge.c @@ -464,6 +464,7 @@ static bool merge_aot_generate(void) char *curr_name = aot_x86_lib_names; uint8_t aot_file_type = get_file_type(merge_seg_info_vector[0]->file_name); p_header->aot_file_type = aot_file_type; + p_header->static_helper = !!option_static_helper; int page_index = 0; for (int i = 0; i < seg_info_num; i++) { seg_info *curr_seg_info = merge_seg_info_vector[i]->s_info; @@ -744,7 +745,8 @@ static AOTLoadResult aot_load_no_lock(char *lib_name) fclose(pf); goto load_error; } - if ((size_t)file_end < strlen(AOT_VERSION)) { + if ((size_t)file_end < sizeof(aot_header) || + (size_t)file_end < strlen(AOT_VERSION)) { fclose(pf); if (i == 0) { goto invalid_base; @@ -788,6 +790,18 @@ static AOTLoadResult aot_load_no_lock(char *lib_name) fclose(pf); goto load_error; } + if (((aot_header *)buffer)->static_helper != + !!option_static_helper) { + qemu_log_mask(LAT_LOG_AOT, + "static helper mode changed, reject aot %s\n", + path); + free(buffer); + fclose(pf); + if (i == 0) { + goto invalid_base; + } + goto load_error; + } fclose(pf); aot_buffer_all[j].p = buffer; aot_buffer_all[j].maplen = file_sz; diff --git a/target/i386/latx/sbt/aot_reader.c b/target/i386/latx/sbt/aot_reader.c index 2273bc84fb0..37de0d3a4e0 100644 --- a/target/i386/latx/sbt/aot_reader.c +++ b/target/i386/latx/sbt/aot_reader.c @@ -8,6 +8,7 @@ #include "aot.h" #include "aot_reader.h" #include "file_ctx.h" +#include "latx-options.h" #include "qemu.h" #ifdef CONFIG_LATX_AOT @@ -65,6 +66,14 @@ int aot_get_tb_num(char *lib_name, char *aot_file_name, CPUState *cpu) aot_header *p_header = (aot_header *)buffer; struct stat statbuf; + if (p_header->static_helper != !!option_static_helper) { + qemu_log_mask(LAT_LOG_AOT, + "static helper mode changed, remove aot %s\n", + aot_file_path); + remove(aot_file_path); + goto out; + } + if ((p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE)) && (stat(lib_name, &statbuf) || p_header->lib_size != statbuf.st_size @@ -157,6 +166,14 @@ lib_info *aot_load(char *lib_name, char *aot_file_name, assert(buffer); aot_header *p_header = (aot_header *)buffer; + if (p_header->static_helper != !!option_static_helper) { + qemu_log_mask(LAT_LOG_AOT, + "static helper mode changed, remove aot %s\n", + aot_file_path); + remove_curr_aot_file(fd); + goto out; + } + if (p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE)) { if (stat(lib_name, &statbuf) || p_header->lib_size != statbuf.st_size diff --git a/target/i386/latx/sbt/tests/aot-cache-reader-test.c b/target/i386/latx/sbt/tests/aot-cache-reader-test.c index f78d1df77b7..4aed21c7c36 100644 --- a/target/i386/latx/sbt/tests/aot-cache-reader-test.c +++ b/target/i386/latx/sbt/tests/aot-cache-reader-test.c @@ -12,6 +12,7 @@ char aot_file_lock_buffer[PATH_MAX]; char *aot_file_path = aot_file_path_buffer; char *aot_file_lock = aot_file_lock_buffer; int qemu_loglevel; +int option_static_helper = 1; static bool fail_fdopen; static int tracked_fd; static int sentinel_fd; @@ -102,6 +103,7 @@ static void write_cache(const char *name, bool has_header, bool has_footer, contents = g_malloc0(size); if (has_header) { ((aot_header *)contents)->aot_file_type = CACHE_AOT_FILE; + ((aot_header *)contents)->static_helper = !!option_static_helper; } if (has_footer) { memcpy(contents + size - footer_size, AOT_VERSION, footer_size); @@ -134,6 +136,7 @@ int main(void) char bad_footer_name[] = "bad-footer"; char truncated_name[] = "truncated"; char complete_name[] = "complete"; + char mode_mismatch_name[] = "mode-mismatch"; char fdopen_failure_name[] = "fdopen-failure"; char cache_path[PATH_MAX]; void *buffer; @@ -171,6 +174,20 @@ int main(void) assert_stream_closed(); g_assert(lib_tree_remove(complete_name)); + option_static_helper = 0; + write_cache(mode_mismatch_name, true, true, cache_path); + option_static_helper = 1; + g_assert(aot_get_tb_num(lib_name, mode_mismatch_name, NULL) == 0); + g_assert(!g_file_test(cache_path, G_FILE_TEST_EXISTS)); + + buffer = NULL; + write_cache(mode_mismatch_name, true, true, cache_path); + option_static_helper = 0; + g_assert(aot_load(lib_name, mode_mismatch_name, &buffer) == NULL); + g_assert(buffer == NULL); + g_assert(!g_file_test(cache_path, G_FILE_TEST_EXISTS)); + option_static_helper = 1; + reset_stream_counts(); fail_fdopen = true; buffer = NULL; @@ -184,6 +201,7 @@ int main(void) remove_cache(bad_footer_name); remove_cache(truncated_name); remove_cache(complete_name); + remove_cache(mode_mismatch_name); remove_cache(fdopen_failure_name); cache_dir = g_build_filename(test_dir, ".cache", "latx", NULL); g_assert(g_rmdir(cache_dir) == 0);