diff --git a/linux-user/main.c b/linux-user/main.c index 5e2541a1942..b8692c1fe96 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -723,6 +723,60 @@ static void handle_arg_latx_imm_reg(const char *arg) options_parse_imm_reg(arg); } +static void handle_arg_latx_imm_rip(const char *arg) +{ + int value; + + if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) { + error_report("LATX_IMM_RIP must be exactly 0 or 1 (got '%s')", arg); + exit(EXIT_FAILURE); + } + option_imm_rip = value; + if (value) { + option_imm_reg = 1; + } +} + +static void handle_arg_latx_imm_rip_stats(const char *arg) +{ + int value; + + if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) { + error_report("LATX_IMM_RIP_STATS must be exactly 0 or 1 (got '%s')", + arg); + exit(EXIT_FAILURE); + } + option_imm_rip_stats = value; +} + +static void handle_arg_latx_imm_complex(const char *arg) +{ + int value; + + if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || + value > LATX_IMM_COMPLEX_ALL) { + error_report("LATX_IMM_COMPLEX must be a mask from 0 to %d (got '%s')", + LATX_IMM_COMPLEX_ALL, arg); + exit(EXIT_FAILURE); + } + option_imm_complex = value; + if (value) { + option_imm_reg = 1; + } +} + +static void handle_arg_latx_imm_complex_stats(const char *arg) +{ + int value; + + if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) { + error_report("LATX_IMM_COMPLEX_STATS must be exactly 0 or 1 " + "(got '%s')", arg); + exit(EXIT_FAILURE); + } + option_imm_complex_stats = value; +} + static void handle_arg_save_xmm(const char *arg) { if (strtol(arg, NULL, 0)) { @@ -945,6 +999,17 @@ static const struct qemu_argument arg_table[] = { "", "enable jrra"}, {"latx-imm-reg", "LATX_IMM_REG", true, handle_arg_latx_imm_reg, "", "enable imm reg optimization"}, + {"latx-imm-rip", "LATX_IMM_RIP", true, handle_arg_latx_imm_rip, + "0|1", "enable CODE64 RIP-relative immediate cache"}, + {"latx-imm-rip-stats", "LATX_IMM_RIP_STATS", true, + handle_arg_latx_imm_rip_stats, "0|1", + "report per-TB RIP-relative immediate cache statistics"}, + {"latx-imm-complex", "LATX_IMM_COMPLEX", true, + handle_arg_latx_imm_complex, "mask", + "enable complex-address cache modes (1=base, 2=index, 4=combined)"}, + {"latx-imm-complex-stats", "LATX_IMM_COMPLEX_STATS", true, + handle_arg_latx_imm_complex_stats, "0|1", + "report per-TB complex-address cache statistics"}, {"latx-mem-test", "LATX_MT", true, handle_arg_latx_mem_test, "", "test memory right when memory access"}, {"latx-real-maps", "LATX_REAL_MAPS", true, handle_arg_latx_real_maps, diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index f72613a814d..cc0ad93947d 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -39,9 +39,9 @@ extern const char *aot_left_file_minsize_optarg; * +--------------+ */ #ifdef CONFIG_LATX_DEBUG -#define AOT_VERSION "Version: "LATX_VERSION"-debug" +#define AOT_VERSION "Version: "LATX_VERSION"-debug-immcfg2" #else -#define AOT_VERSION "Version: "LATX_VERSION"-release" +#define AOT_VERSION "Version: "LATX_VERSION"-release-immcfg2" #endif typedef struct aot_header { uint32_t lib_size; @@ -57,6 +57,8 @@ typedef struct aot_header { #define CACHE_AOT_FILE 4 #define HASH_AOT_FILE 8 uint8_t aot_file_type; + uint8_t imm_rip; + uint8_t imm_complex; } aot_header; typedef struct aot_file_info { diff --git a/target/i386/latx/include/imm-cache.h b/target/i386/latx/include/imm-cache.h index 52ca417980e..419c3a8250e 100644 --- a/target/i386/latx/include/imm-cache.h +++ b/target/i386/latx/include/imm-cache.h @@ -19,6 +19,7 @@ #define CACHE_DEFAULT_CAPACITY 4 #define CACHE_MAX_CAPACITY 300 +#define IMM_CACHE_RIP_BASE (-100) #define imm_log(FMT, ...) \ do { \ @@ -61,6 +62,54 @@ typedef struct { int64 max_offset; } IMM_CACHE_BUCKET; +typedef struct { + uint64_t calls; + uint64_t hits; + uint64_t misses; + uint64_t replacements; + uint64_t itemp_fallbacks; + uint64_t direct_hits; + uint64_t host_offset_hits; + uint64_t addi_hits; + uint64_t full_loads; + uint64_t helper_invalidations; +} IMM_CACHE_RIP_STATS; + +typedef enum { + IMM_CACHE_COMPLEX_BASE_DISP, + IMM_CACHE_COMPLEX_INDEX_DISP, + IMM_CACHE_COMPLEX_BASE_INDEX_DISP, + IMM_CACHE_COMPLEX_KIND_COUNT, +} IMM_CACHE_COMPLEX_KIND; + +typedef enum { + IMM_CACHE_COMPLEX_SKIP_DISABLED, + IMM_CACHE_COMPLEX_SKIP_NOT_CODE64, + IMM_CACHE_COMPLEX_SKIP_ADDR_SIZE, + IMM_CACHE_COMPLEX_SKIP_SEGMENT, + IMM_CACHE_COMPLEX_SKIP_UNSUPPORTED_FORM, + IMM_CACHE_COMPLEX_SKIP_INVALID_SCALE, + IMM_CACHE_COMPLEX_SKIP_NO_BENEFIT, + IMM_CACHE_COMPLEX_SKIP_INSTRUCTION, + IMM_CACHE_COMPLEX_SKIP_FIXED_DEST, + IMM_CACHE_COMPLEX_SKIP_COUNT, +} IMM_CACHE_COMPLEX_SKIP_REASON; + +typedef struct { + uint64_t calls[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t hits[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t misses[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t replacements[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t itemp_conflicts[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t direct_hits[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t host_offset_hits[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t addi_hits[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t full_generations[IMM_CACHE_COMPLEX_KIND_COUNT]; + uint64_t skips[IMM_CACHE_COMPLEX_SKIP_COUNT]; + uint64_t base_index_invalidations; + uint64_t use_def_fallbacks; +} IMM_CACHE_COMPLEX_STATS; + /** * imm format: * (offset can be 0) @@ -93,6 +142,8 @@ typedef struct { * if we use cached itemp, ra_free_temp should skip */ bool itemp_allocated; + IMM_CACHE_RIP_STATS rip_stats; + IMM_CACHE_COMPLEX_STATS complex_stats; /** * for complex imm, we need to compare whether base/index reg has been @@ -156,6 +207,9 @@ void imm_cache_free_itemp(IMM_CACHE *cache, int itemp_num); void imm_cache_free_dead_cache(IMM_CACHE *cache); void imm_cache_free(IMM_CACHE *cache, int id); void imm_cache_free_all(IMM_CACHE *cache); +void imm_cache_invalidate_for_helper(void); +void imm_cache_print_rip_stats(IMM_CACHE *cache, uint64_t tb_pc); +void imm_cache_print_complex_stats(IMM_CACHE *cache, uint64_t tb_pc); bool imm_cache_is_imm_itemp(int itemp_num); bool imm_cache_itemp_is_used(IR2_OPND opnd); void imm_cache_free_cache_use_target_reg(IR2_OPND opnd); diff --git a/target/i386/latx/include/latx-options.h b/target/i386/latx/include/latx-options.h index fce461a7f47..813a8ad754d 100644 --- a/target/i386/latx/include/latx-options.h +++ b/target/i386/latx/include/latx-options.h @@ -118,7 +118,13 @@ extern int option_anonym; extern int option_imm_reg; extern int option_imm_precache; extern int option_imm_rip; +extern int option_imm_rip_stats; +#define LATX_IMM_COMPLEX_BASE_DISP (1 << 0) +#define LATX_IMM_COMPLEX_INDEX_DISP (1 << 1) +#define LATX_IMM_COMPLEX_BASE_INDEX_DISP (1 << 2) +#define LATX_IMM_COMPLEX_ALL ((1 << 3) - 1) extern int option_imm_complex; +extern int option_imm_complex_stats; extern int option_debug_imm_reg; extern uint64_t imm_skip_pc; extern int option_mem_test; @@ -160,6 +166,10 @@ extern unsigned long long counter_mips_tr; ENVFUN(SAVE_XMM, handle_arg_save_xmm) \ ENVFUN(LATX_JRRA, handle_arg_latx_jrra) \ ENVFUN(LATX_IMM_REG, handle_arg_latx_imm_reg) \ + ENVFUN(LATX_IMM_RIP, handle_arg_latx_imm_rip) \ + ENVFUN(LATX_IMM_RIP_STATS, handle_arg_latx_imm_rip_stats) \ + ENVFUN(LATX_IMM_COMPLEX, handle_arg_latx_imm_complex) \ + ENVFUN(LATX_IMM_COMPLEX_STATS, handle_arg_latx_imm_complex_stats) \ ENVFUN(LATX_MT, handle_arg_latx_mem_test) \ ENVFUN(LATX_REAL_MAPS, handle_arg_latx_real_maps) \ ENVFUN(LATX_MONITOR_SHARED_MEM, handle_arg_latx_monitor_shared_mem) \ diff --git a/target/i386/latx/latx-options.c b/target/i386/latx/latx-options.c index 43671bbb556..b139bde773a 100644 --- a/target/i386/latx/latx-options.c +++ b/target/i386/latx/latx-options.c @@ -81,8 +81,10 @@ int option_smc_reload; int option_debug_aot; int option_imm_reg; int option_imm_rip; +int option_imm_rip_stats; int option_imm_precache; int option_imm_complex; +int option_imm_complex_stats; int option_debug_imm_reg; uint64_t imm_skip_pc; uint64_t debug_tb_pc; @@ -255,10 +257,16 @@ void options_init(void) #endif option_smc_reload = 0; #ifdef CONFIG_LATX_IMM_REG +#ifdef TARGET_X86_64 + option_imm_reg = 1; + option_imm_complex = LATX_IMM_COMPLEX_ALL; +#else option_imm_reg = 0; - option_imm_rip = 0; - // complex:base+index*scale option_imm_complex = 0; +#endif + option_imm_rip = 0; + option_imm_rip_stats = 0; + option_imm_complex_stats = 0; option_imm_precache = 0; option_debug_imm_reg = 0; imm_skip_pc = 0; @@ -342,7 +350,7 @@ void options_parse_imm_reg(const char *bits) } if (bits[OPTIONS_IMM_COMPLEX] == '1') { - option_imm_complex = 1; + option_imm_complex = LATX_IMM_COMPLEX_ALL; } else if (bits[OPTIONS_IMM_COMPLEX] == '0') { option_imm_complex = 0; } else { diff --git a/target/i386/latx/optimization/imm-cache.c b/target/i386/latx/optimization/imm-cache.c index dffefbbada4..145296f03b0 100644 --- a/target/i386/latx/optimization/imm-cache.c +++ b/target/i386/latx/optimization/imm-cache.c @@ -174,6 +174,8 @@ void imm_cache_init(IMM_CACHE *imm_cache, int capacity) imm_cache->curr_ir2_index = 0; imm_cache->optimized_ir2 = false; imm_cache->itemp_allocated = false; + memset(&imm_cache->rip_stats, 0, sizeof(imm_cache->rip_stats)); + memset(&imm_cache->complex_stats, 0, sizeof(imm_cache->complex_stats)); for (int i = 0; i < 16; i++) { imm_cache->ir1_reg_last_updated_index[i] = -1; } @@ -219,62 +221,194 @@ void imm_cache_fill_bucket(IMM_CACHE *cache, int cache_id, int base, int index, cache->bucket[cache_id].use_count = 0; } -/** - * TODO: some instructions will change gpr indirectly - */ +static void imm_cache_mark_gpr_updated(IMM_CACHE *cache, int reg_num, + int curr_ir1_index) +{ + if (reg_num >= 0 && reg_num < 16) { + cache->ir1_reg_last_updated_index[reg_num] = curr_ir1_index; + } +} + void imm_cache_update_ir1_usage(IMM_CACHE *cache, IR1_INST *pir1, int curr_ir1_index) { - /* trace x86_reg latest define value */ - if (pir1->info->x86.op_count > 0) { - IR1_OPCODE opcode = ir1_opcode(pir1); - int *ir1_updated_index = cache->ir1_reg_last_updated_index; - switch (opcode) { - case dt_X86_INS_PUSH: - case dt_X86_INS_PUSHF: - case dt_X86_INS_PUSHFD: - case dt_X86_INS_PUSHAL: - case dt_X86_INS_PUSHAW: - - case dt_X86_INS_POP: - case dt_X86_INS_POPF: - ir1_updated_index[esp_index] = curr_ir1_index; - break; - case dt_X86_INS_MUL: - case dt_X86_INS_IMUL: - case dt_X86_INS_DIV: - case dt_X86_INS_IDIV: - ir1_updated_index[eax_index] = curr_ir1_index; - ir1_updated_index[edx_index] = curr_ir1_index; - break; - // both src and dest will change - case dt_X86_INS_XCHG: - case dt_X86_INS_XADD: - for (int i = 0; i < 2; i++) { - IR1_OPND *opnd0 = ir1_get_opnd(pir1, i); - if (ir1_opnd_is_gpr(opnd0)) { - int reg_num = ir1_opnd_base_reg_num(opnd0); - ir1_updated_index[reg_num] = curr_ir1_index; - } - return; - } - break; - // all reg changes - case dt_X86_INS_POPAL: - case dt_X86_INS_POPAW: - for (int i = 0; i < 16; i++) { - ir1_updated_index[i] = curr_ir1_index; - } - break; - default: - break; + IR1_OPCODE opcode = ir1_opcode(pir1); + int op_count = pir1->info->x86.op_count; + bool string_op = false; + + for (int i = 0; i < op_count; i++) { + IR1_OPND *opnd = ir1_get_opnd(pir1, i); + + if (ir1_opnd_is_gpr(opnd) && + (opnd->access & dt_CS_AC_WRITE)) { + imm_cache_mark_gpr_updated(cache, ir1_opnd_base_reg_num(opnd), + curr_ir1_index); } + } + + /* Preserve the historical operand-zero fallback for incomplete metadata. */ + if (op_count > 0) { IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); - if (ir1_opnd_is_gpr(opnd0)) { - int reg_num = ir1_opnd_base_reg_num(opnd0); - ir1_updated_index[reg_num] = curr_ir1_index; + + if (ir1_opnd_is_gpr(opnd0) && + opnd0->access == dt_CS_AC_INVALID) { + imm_cache_mark_gpr_updated(cache, + ir1_opnd_base_reg_num(opnd0), + curr_ir1_index); + cache->complex_stats.use_def_fallbacks++; } } + + /* + * CAPSTONE_DIET omits implicit register metadata. Conservatively treat + * operand-less instructions as clobbering every GPR so an unlisted hidden + * destination can only reduce cache reuse, never leave a stale address. + */ + if (option_imm_complex && op_count == 0) { + for (int i = 0; i < 16; i++) { + imm_cache_mark_gpr_updated(cache, i, curr_ir1_index); + } + } + + switch (opcode) { + case dt_X86_INS_PUSH: + case dt_X86_INS_PUSHF: + case dt_X86_INS_PUSHFD: + case dt_X86_INS_PUSHFQ: + case dt_X86_INS_PUSHAL: + case dt_X86_INS_PUSHAW: + case dt_X86_INS_POP: + case dt_X86_INS_POPF: + case dt_X86_INS_POPFD: + case dt_X86_INS_POPFQ: + imm_cache_mark_gpr_updated(cache, esp_index, curr_ir1_index); + break; + case dt_X86_INS_ENTER: + case dt_X86_INS_LEAVE: + imm_cache_mark_gpr_updated(cache, esp_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, ebp_index, curr_ir1_index); + break; + case dt_X86_INS_MUL: + case dt_X86_INS_DIV: + case dt_X86_INS_IDIV: + imm_cache_mark_gpr_updated(cache, eax_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, edx_index, curr_ir1_index); + break; + case dt_X86_INS_IMUL: + if (op_count == 1) { + imm_cache_mark_gpr_updated(cache, eax_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, edx_index, curr_ir1_index); + } + break; + case dt_X86_INS_CBW: + case dt_X86_INS_CWDE: + case dt_X86_INS_CDQE: + case dt_X86_INS_LAHF: + case dt_X86_INS_SALC: + case dt_X86_INS_IN: + case dt_X86_INS_XLATB: + imm_cache_mark_gpr_updated(cache, eax_index, curr_ir1_index); + break; + case dt_X86_INS_CWD: + case dt_X86_INS_CDQ: + case dt_X86_INS_CQO: + imm_cache_mark_gpr_updated(cache, edx_index, curr_ir1_index); + break; + case dt_X86_INS_CMPXCHG: + imm_cache_mark_gpr_updated(cache, eax_index, curr_ir1_index); + break; + case dt_X86_INS_CMPXCHG8B: + case dt_X86_INS_CMPXCHG16B: + case dt_X86_INS_RDTSC: + case dt_X86_INS_XGETBV: + imm_cache_mark_gpr_updated(cache, eax_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, edx_index, curr_ir1_index); + break; + case dt_X86_INS_RDTSCP: + imm_cache_mark_gpr_updated(cache, eax_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, ecx_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, edx_index, curr_ir1_index); + break; + case dt_X86_INS_CPUID: + imm_cache_mark_gpr_updated(cache, eax_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, ebx_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, ecx_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, edx_index, curr_ir1_index); + break; + case dt_X86_INS_LODSB: + case dt_X86_INS_LODSW: + case dt_X86_INS_LODSD: + case dt_X86_INS_LODSQ: + imm_cache_mark_gpr_updated(cache, eax_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, esi_index, curr_ir1_index); + string_op = true; + break; + case dt_X86_INS_STOSB: + case dt_X86_INS_STOSW: + case dt_X86_INS_STOSD: + case dt_X86_INS_STOSQ: + case dt_X86_INS_SCASB: + case dt_X86_INS_SCASW: + case dt_X86_INS_SCASD: + case dt_X86_INS_SCASQ: + case dt_X86_INS_INSB: + case dt_X86_INS_INSW: + case dt_X86_INS_INSD: + imm_cache_mark_gpr_updated(cache, edi_index, curr_ir1_index); + string_op = true; + break; + case dt_X86_INS_MOVSB: + case dt_X86_INS_MOVSW: + case dt_X86_INS_MOVSQ: + case dt_X86_INS_CMPSB: + case dt_X86_INS_CMPSW: + case dt_X86_INS_CMPSQ: + imm_cache_mark_gpr_updated(cache, esi_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, edi_index, curr_ir1_index); + string_op = true; + break; + case dt_X86_INS_MOVSD: + if (pir1->info->x86.opcode[0] == 0xa5) { + imm_cache_mark_gpr_updated(cache, esi_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, edi_index, curr_ir1_index); + string_op = true; + } + break; + case dt_X86_INS_CMPSD: + if (pir1->info->x86.opcode[0] == 0xa7) { + imm_cache_mark_gpr_updated(cache, esi_index, curr_ir1_index); + imm_cache_mark_gpr_updated(cache, edi_index, curr_ir1_index); + string_op = true; + } + break; + case dt_X86_INS_XCHG: + case dt_X86_INS_XADD: + case dt_X86_INS_MULX: + /* These instructions have two explicit destination operands. */ + for (int i = 0; i < MIN(op_count, 2); i++) { + IR1_OPND *opnd = ir1_get_opnd(pir1, i); + + if (ir1_opnd_is_gpr(opnd)) { + imm_cache_mark_gpr_updated( + cache, ir1_opnd_base_reg_num(opnd), curr_ir1_index); + } + } + break; + case dt_X86_INS_POPAL: + case dt_X86_INS_POPAW: + for (int i = 0; i < 16; i++) { + imm_cache_mark_gpr_updated(cache, i, curr_ir1_index); + } + break; + default: + break; + } + + if (string_op && + (ir1_prefix(pir1) == dt_X86_PREFIX_REP || + ir1_prefix(pir1) == dt_X86_PREFIX_REPNE)) { + imm_cache_mark_gpr_updated(cache, ecx_index, curr_ir1_index); + } } void imm_cache_check_ir1_should_skip(bool *bool_skip_ptr) @@ -478,7 +612,24 @@ IMM_CACHE_RES imm_cache_allocate(IMM_CACHE *cache, int base, int index, stop_opt = true; } } - const char *prefix = (base == -100) ? "rip" : "complex"; + bool is_rip = base == IMM_CACHE_RIP_BASE; + int complex_kind = -1; + + if (!is_rip) { + if (base >= 0 && index >= 0) { + complex_kind = IMM_CACHE_COMPLEX_BASE_INDEX_DISP; + } else if (base >= 0) { + complex_kind = IMM_CACHE_COMPLEX_BASE_DISP; + } else if (index >= 0) { + complex_kind = IMM_CACHE_COMPLEX_INDEX_DISP; + } + } + const char *prefix = is_rip ? "rip" : "complex"; + if (is_rip) { + cache->rip_stats.calls++; + } else if (complex_kind >= 0) { + cache->complex_stats.calls[complex_kind]++; + } imm_log("[allocate %s]>>>>>>>>>> start >>>>>>>>>>>\n" "[allocate %s] b:%d\ti:%d\ts:%d\toffset:0x%lx\n", prefix, prefix, base, index, scale, offset); @@ -489,8 +640,14 @@ IMM_CACHE_RES imm_cache_allocate(IMM_CACHE *cache, int base, int index, // dead cache auto free // imm_cache_free_dead_cache(cache); - IMM_CACHE_RES res; - res.pre_cache = false; + IMM_CACHE_RES res = { + .diff = 0, + .offset = offset, + .itemp_num = -1, + .cache_id = -1, + .pre_cache = false, + .cached = false, + }; int cache_id = -1; if (likely(!stop_opt)) { @@ -499,6 +656,17 @@ IMM_CACHE_RES imm_cache_allocate(IMM_CACHE *cache, int base, int index, if (cache_id == -1) { // a new cache imm_log("[new]\t\n"); + if (is_rip) { + cache->rip_stats.misses++; + if (cache->free_count == 0) { + cache->rip_stats.replacements++; + } + } else if (complex_kind >= 0) { + cache->complex_stats.misses[complex_kind]++; + if (cache->free_count == 0) { + cache->complex_stats.replacements[complex_kind]++; + } + } cache_id = imm_cache_put(cache, base, index, scale, offset); res.cached = false; } else { @@ -518,6 +686,11 @@ IMM_CACHE_RES imm_cache_allocate(IMM_CACHE *cache, int base, int index, res.cached = true; cache->bucket[cache_id].use_count++; cache_hit++; + if (is_rip) { + cache->rip_stats.hits++; + } else if (complex_kind >= 0) { + cache->complex_stats.hits[complex_kind]++; + } } else { // repace cache imm_log("[cache cannot use] replace cache_id:%d\n", cache_id); @@ -526,6 +699,14 @@ IMM_CACHE_RES imm_cache_allocate(IMM_CACHE *cache, int base, int index, offset); res.offset = offset; res.cached = false; + if (is_rip) { + cache->rip_stats.misses++; + cache->rip_stats.replacements++; + } else if (complex_kind >= 0) { + cache->complex_stats.misses[complex_kind]++; + cache->complex_stats.replacements[complex_kind]++; + cache->complex_stats.base_index_invalidations++; + } // imm_cache_sort(cache); } @@ -570,10 +751,14 @@ long imm_cache_diff(IMM_CACHE *cache, int cache_id, int64 new_offset) void imm_cache_update_by_diff(IMM_CACHE *cache, int cache_id, long diff) { - int64 old_offset = cache->bucket[cache_id].offset; + IMM_CACHE_BUCKET *bucket = &cache->bucket[cache_id]; + int64 old_offset = + bucket->avg_offset == 0 ? bucket->offset : bucket->avg_offset; + qemu_log_mask(LAT_IMM_REG, "[update by diff]new:%lx+%lx=%lx\n", old_offset, diff, old_offset + diff); - cache->bucket[cache_id].offset += diff; + bucket->offset = old_offset + diff; + bucket->avg_offset = 0; } void imm_cache_update_by_offset(IMM_CACHE *cache, int cache_id, @@ -663,10 +848,121 @@ void imm_cache_free(IMM_CACHE *cache, int id) void imm_cache_free_all(IMM_CACHE *cache) { + if (!cache) { + return; + } for (int i = 0; i < cache->cache_count; i++) { - imm_cache_free(cache, i); + if (!cache->bucket[i].free) { + imm_cache_free(cache, i); + } } cache->free_count = cache->cache_count; + cache->itemp_allocated = false; +} + +void imm_cache_invalidate_for_helper(void) +{ + IMM_CACHE *cache; + bool has_rip = false; + + if (!option_imm_reg || !lsenv || !lsenv->tr_data) { + return; + } + cache = lsenv->tr_data->imm_cache; + if (!cache) { + return; + } + for (int i = 0; i < cache->cache_count; i++) { + if (!cache->bucket[i].free && + cache->bucket[i].base == IMM_CACHE_RIP_BASE) { + has_rip = true; + break; + } + } + if (has_rip) { + cache->rip_stats.helper_invalidations++; + } + imm_cache_free_all(cache); +} + +void imm_cache_print_rip_stats(IMM_CACHE *cache, uint64_t tb_pc) +{ + IMM_CACHE_RIP_STATS *stats; + + if (!option_imm_rip_stats || !cache) { + return; + } + stats = &cache->rip_stats; + if (!stats->calls) { + return; + } + fprintf(stderr, + "[LATX][imm-rip] tb=0x%" PRIx64 + " calls=%" PRIu64 " hits=%" PRIu64 " misses=%" PRIu64 + " replacements=%" PRIu64 " itemp-fallbacks=%" PRIu64 + " direct-hits=%" PRIu64 " host-offset-hits=%" PRIu64 + " addi-hits=%" PRIu64 " full-loads=%" PRIu64 + " helper-invalidations=%" PRIu64 "\n", + tb_pc, stats->calls, stats->hits, stats->misses, + stats->replacements, stats->itemp_fallbacks, + stats->direct_hits, stats->host_offset_hits, stats->addi_hits, + stats->full_loads, stats->helper_invalidations); +} + +void imm_cache_print_complex_stats(IMM_CACHE *cache, uint64_t tb_pc) +{ + static const char * const kind_names[] = { + "base-disp", "index-disp", "base-index-disp", + }; + IMM_CACHE_COMPLEX_STATS *stats; + uint64_t activity = 0; + + if (!option_imm_complex_stats || !cache) { + return; + } + stats = &cache->complex_stats; + for (int i = 0; i < IMM_CACHE_COMPLEX_KIND_COUNT; i++) { + activity += stats->calls[i]; + } + for (int i = 0; i < IMM_CACHE_COMPLEX_SKIP_COUNT; i++) { + activity += stats->skips[i]; + } + if (!activity) { + return; + } + + for (int i = 0; i < IMM_CACHE_COMPLEX_KIND_COUNT; i++) { + fprintf(stderr, + "[LATX][imm-complex] tb=0x%" PRIx64 " mode=%s" + " calls=%" PRIu64 " hits=%" PRIu64 " misses=%" PRIu64 + " replacements=%" PRIu64 " itemp-conflicts=%" PRIu64 + " direct-hits=%" PRIu64 " host-offset-hits=%" PRIu64 + " addi-hits=%" PRIu64 " full-generations=%" PRIu64 "\n", + tb_pc, kind_names[i], stats->calls[i], stats->hits[i], + stats->misses[i], stats->replacements[i], + stats->itemp_conflicts[i], stats->direct_hits[i], + stats->host_offset_hits[i], stats->addi_hits[i], + stats->full_generations[i]); + } + fprintf(stderr, + "[LATX][imm-complex-skip] tb=0x%" PRIx64 + " disabled=%" PRIu64 " not-code64=%" PRIu64 + " addr-size=%" PRIu64 " segment=%" PRIu64 + " unsupported-form=%" PRIu64 " invalid-scale=%" PRIu64 + " no-benefit=%" PRIu64 " instruction=%" PRIu64 + " fixed-dest=%" PRIu64 " base-index-invalidations=%" PRIu64 + " use-def-fallbacks=%" PRIu64 "\n", + tb_pc, + stats->skips[IMM_CACHE_COMPLEX_SKIP_DISABLED], + stats->skips[IMM_CACHE_COMPLEX_SKIP_NOT_CODE64], + stats->skips[IMM_CACHE_COMPLEX_SKIP_ADDR_SIZE], + stats->skips[IMM_CACHE_COMPLEX_SKIP_SEGMENT], + stats->skips[IMM_CACHE_COMPLEX_SKIP_UNSUPPORTED_FORM], + stats->skips[IMM_CACHE_COMPLEX_SKIP_INVALID_SCALE], + stats->skips[IMM_CACHE_COMPLEX_SKIP_NO_BENEFIT], + stats->skips[IMM_CACHE_COMPLEX_SKIP_INSTRUCTION], + stats->skips[IMM_CACHE_COMPLEX_SKIP_FIXED_DEST], + stats->base_index_invalidations, stats->use_def_fallbacks); } void imm_cache_print_bucket(IMM_CACHE_BUCKET *bucket, int index) diff --git a/target/i386/latx/sbt/aot.c b/target/i386/latx/sbt/aot.c index 3aa6292cfac..ce44c49aaff 100644 --- a/target/i386/latx/sbt/aot.c +++ b/target/i386/latx/sbt/aot.c @@ -766,6 +766,8 @@ int do_generate_aot(int first_seg_in_lib, int end_seg_in_lib) p_header->aot_file_type = seg_info_vector[first_seg_in_lib]->aot_file_type; + p_header->imm_rip = !!(option_imm_reg && option_imm_rip); + p_header->imm_complex = option_imm_reg ? option_imm_complex : 0; if (p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE)) { struct stat statbuf; diff --git a/target/i386/latx/sbt/aot_merge.c b/target/i386/latx/sbt/aot_merge.c index 16ce5cdf721..7321fd48753 100644 --- a/target/i386/latx/sbt/aot_merge.c +++ b/target/i386/latx/sbt/aot_merge.c @@ -464,6 +464,8 @@ static bool merge_aot_generate(void) char *curr_name = aot_x86_lib_names; uint8_t aot_file_type = get_file_type(merge_seg_info_vector[0]->file_name); p_header->aot_file_type = aot_file_type; + p_header->imm_rip = !!(option_imm_reg && option_imm_rip); + p_header->imm_complex = option_imm_reg ? option_imm_complex : 0; int page_index = 0; for (int i = 0; i < seg_info_num; i++) { seg_info *curr_seg_info = merge_seg_info_vector[i]->s_info; @@ -744,7 +746,8 @@ static AOTLoadResult aot_load_no_lock(char *lib_name) fclose(pf); goto load_error; } - if ((size_t)file_end < strlen(AOT_VERSION)) { + if ((size_t)file_end < sizeof(aot_header) || + (size_t)file_end < strlen(AOT_VERSION)) { fclose(pf); if (i == 0) { goto invalid_base; @@ -788,6 +791,30 @@ static AOTLoadResult aot_load_no_lock(char *lib_name) fclose(pf); goto load_error; } + if (((aot_header *)buffer)->imm_rip != + !!(option_imm_reg && option_imm_rip)) { + qemu_log_mask(LAT_LOG_AOT, + "RIP immediate cache mode changed, reject aot %s\n", + path); + free(buffer); + fclose(pf); + if (i == 0) { + goto invalid_base; + } + goto load_error; + } + if (((aot_header *)buffer)->imm_complex != + (option_imm_reg ? option_imm_complex : 0)) { + qemu_log_mask(LAT_LOG_AOT, + "complex immediate cache mode changed, reject aot " + "%s\n", path); + free(buffer); + fclose(pf); + if (i == 0) { + goto invalid_base; + } + goto load_error; + } fclose(pf); aot_buffer_all[j].p = buffer; aot_buffer_all[j].maplen = file_sz; diff --git a/target/i386/latx/sbt/aot_reader.c b/target/i386/latx/sbt/aot_reader.c index 2273bc84fb0..f6e7955fc44 100644 --- a/target/i386/latx/sbt/aot_reader.c +++ b/target/i386/latx/sbt/aot_reader.c @@ -8,6 +8,7 @@ #include "aot.h" #include "aot_reader.h" #include "file_ctx.h" +#include "latx-options.h" #include "qemu.h" #ifdef CONFIG_LATX_AOT @@ -65,6 +66,22 @@ int aot_get_tb_num(char *lib_name, char *aot_file_name, CPUState *cpu) aot_header *p_header = (aot_header *)buffer; struct stat statbuf; + if (p_header->imm_rip != !!(option_imm_reg && option_imm_rip)) { + qemu_log_mask(LAT_LOG_AOT, + "RIP immediate cache mode changed, remove aot %s\n", + aot_file_path); + remove(aot_file_path); + goto out; + } + if (p_header->imm_complex != + (option_imm_reg ? option_imm_complex : 0)) { + qemu_log_mask(LAT_LOG_AOT, + "complex immediate cache mode changed, remove aot %s\n", + aot_file_path); + remove(aot_file_path); + goto out; + } + if ((p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE)) && (stat(lib_name, &statbuf) || p_header->lib_size != statbuf.st_size @@ -157,6 +174,22 @@ lib_info *aot_load(char *lib_name, char *aot_file_name, assert(buffer); aot_header *p_header = (aot_header *)buffer; + if (p_header->imm_rip != !!(option_imm_reg && option_imm_rip)) { + qemu_log_mask(LAT_LOG_AOT, + "RIP immediate cache mode changed, remove aot %s\n", + aot_file_path); + remove_curr_aot_file(fd); + goto out; + } + if (p_header->imm_complex != + (option_imm_reg ? option_imm_complex : 0)) { + qemu_log_mask(LAT_LOG_AOT, + "complex immediate cache mode changed, remove aot %s\n", + aot_file_path); + remove_curr_aot_file(fd); + goto out; + } + if (p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE)) { if (stat(lib_name, &statbuf) || p_header->lib_size != statbuf.st_size diff --git a/target/i386/latx/sbt/tests/aot-cache-reader-test.c b/target/i386/latx/sbt/tests/aot-cache-reader-test.c index f78d1df77b7..0c79dc81d36 100644 --- a/target/i386/latx/sbt/tests/aot-cache-reader-test.c +++ b/target/i386/latx/sbt/tests/aot-cache-reader-test.c @@ -6,12 +6,16 @@ #include "aot_reader.h" #include "aot_lib.h" #include "file_ctx.h" +#include "latx-options.h" char aot_file_path_buffer[PATH_MAX]; char aot_file_lock_buffer[PATH_MAX]; char *aot_file_path = aot_file_path_buffer; char *aot_file_lock = aot_file_lock_buffer; int qemu_loglevel; +int option_imm_reg; +int option_imm_rip; +int option_imm_complex; static bool fail_fdopen; static int tracked_fd; static int sentinel_fd; @@ -102,6 +106,10 @@ static void write_cache(const char *name, bool has_header, bool has_footer, contents = g_malloc0(size); if (has_header) { ((aot_header *)contents)->aot_file_type = CACHE_AOT_FILE; + ((aot_header *)contents)->imm_rip = + !!(option_imm_reg && option_imm_rip); + ((aot_header *)contents)->imm_complex = + option_imm_reg ? option_imm_complex : 0; } if (has_footer) { memcpy(contents + size - footer_size, AOT_VERSION, footer_size); @@ -134,6 +142,7 @@ int main(void) char bad_footer_name[] = "bad-footer"; char truncated_name[] = "truncated"; char complete_name[] = "complete"; + char mode_mismatch_name[] = "mode-mismatch"; char fdopen_failure_name[] = "fdopen-failure"; char cache_path[PATH_MAX]; void *buffer; @@ -171,6 +180,37 @@ int main(void) assert_stream_closed(); g_assert(lib_tree_remove(complete_name)); + option_imm_reg = 0; + option_imm_rip = 0; + write_cache(mode_mismatch_name, true, true, cache_path); + option_imm_reg = 1; + option_imm_rip = 1; + g_assert(aot_get_tb_num(lib_name, mode_mismatch_name, NULL) == 0); + g_assert(!g_file_test(cache_path, G_FILE_TEST_EXISTS)); + + buffer = NULL; + write_cache(mode_mismatch_name, true, true, cache_path); + option_imm_rip = 0; + g_assert(aot_load(lib_name, mode_mismatch_name, &buffer) == NULL); + g_assert(buffer == NULL); + g_assert(!g_file_test(cache_path, G_FILE_TEST_EXISTS)); + option_imm_reg = 0; + + option_imm_reg = 1; + option_imm_complex = LATX_IMM_COMPLEX_BASE_DISP; + write_cache(mode_mismatch_name, true, true, cache_path); + option_imm_complex = LATX_IMM_COMPLEX_BASE_INDEX_DISP; + g_assert(aot_get_tb_num(lib_name, mode_mismatch_name, NULL) == 0); + g_assert(!g_file_test(cache_path, G_FILE_TEST_EXISTS)); + + buffer = NULL; + write_cache(mode_mismatch_name, true, true, cache_path); + option_imm_complex = 0; + g_assert(aot_load(lib_name, mode_mismatch_name, &buffer) == NULL); + g_assert(buffer == NULL); + g_assert(!g_file_test(cache_path, G_FILE_TEST_EXISTS)); + option_imm_reg = 0; + reset_stream_counts(); fail_fdopen = true; buffer = NULL; @@ -184,6 +224,7 @@ int main(void) remove_cache(bad_footer_name); remove_cache(truncated_name); remove_cache(complete_name); + remove_cache(mode_mismatch_name); remove_cache(fdopen_failure_name); cache_dir = g_build_filename(test_dir, ".cache", "latx", NULL); g_assert(g_rmdir(cache_dir) == 0); diff --git a/target/i386/latx/translator/mem-interface.c b/target/i386/latx/translator/mem-interface.c index 5ad6bdf96c5..776542b908b 100644 --- a/target/i386/latx/translator/mem-interface.c +++ b/target/i386/latx/translator/mem-interface.c @@ -84,6 +84,22 @@ static void index_add_base(IR2_OPND dest, IR2_OPND base, } } +#ifdef CONFIG_LATX_IMM_REG +static int imm_cache_complex_kind(bool has_base, bool has_index, longx offset) +{ + if (has_base && has_index) { + return IMM_CACHE_COMPLEX_BASE_INDEX_DISP; + } + if (has_index) { + return IMM_CACHE_COMPLEX_INDEX_DISP; + } + if (has_base && offset) { + return IMM_CACHE_COMPLEX_BASE_DISP; + } + return -1; +} +#endif + /** * @brief adjust_dest - store the dest based on the values of * dest_size and addr_size. @@ -181,6 +197,9 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, IMM_CACHE *imm_cache = lsenv->tr_data->imm_cache; imm_cache->itemp_allocated = false; bool cache_skip = false; + bool complex_attempted = false; + int complex_kind = -1; + int complex_host_off = 0; #endif dest_need_itmp = false; if (!arg_dest_op) { @@ -210,6 +229,7 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, pir1 = lsenv->tr_data->curr_ir1_inst; lsassert(pir1 != NULL); + addr_size = ir1_addr_size(pir1); offset = ir1_opnd_simm(opnd1); TranslationBlock *tb __attribute__((unused)) = lsenv->tr_data->curr_tb; @@ -228,14 +248,16 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, #ifdef CONFIG_LATX_IMM_REG if (dest_need_itmp) { - if (!option_imm_reg || !option_imm_rip || cache_skip) { + if (!option_imm_reg || !option_imm_rip || cache_skip || + addr_size != 64 || ir1_opnd_has_seg(opnd1)) { dest_op = ra_alloc_itemp(); imm_cache->itemp_allocated = true; } else { /* Determine if this RIP address can be optimized */ IMM_CACHE_RES res = - imm_cache_allocate(imm_cache, -100, -1, -1, offset); + imm_cache_allocate(imm_cache, IMM_CACHE_RIP_BASE, + -1, -1, offset); dest_op = ra_alloc_imm_reg(res.itemp_num); if (ir2_opnd_is_none(&dest_op)) { // itemp is not available skip directly @@ -243,6 +265,7 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, imm_log("[alloc none] %d\n", res.itemp_num); free_imm_reg(res.itemp_num); imm_cache->itemp_allocated = true; + imm_cache->rip_stats.itemp_fallbacks++; } else { // 1. cached? if (res.cached) { @@ -273,11 +296,15 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, } if (host_off) { *host_off = res.diff; + imm_cache->rip_stats.host_offset_hits++; } else if (res.diff != 0) { // cache imm offset should be updated at the sametime if no host_off imm_cache_update_by_diff(imm_cache, res.cache_id, res.diff); la_addi_d(dest_op, dest_op, res.diff); + imm_cache->rip_stats.addi_hits++; + } else { + imm_cache->rip_stats.direct_hits++; } return dest_op; } @@ -292,6 +319,12 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, target_ulong call_offset __attribute__((unused)) = aot_get_call_offset(offset); aot_load_guest_addr(dest_op, offset, LOAD_CALL_TARGET, call_offset); +#ifdef CONFIG_LATX_IMM_REG + if (option_imm_reg && option_imm_rip && addr_size == 64 && + !ir1_opnd_has_seg(opnd1)) { + imm_cache->rip_stats.full_loads++; + } +#endif if (host_off) { *host_off = 0; } @@ -319,9 +352,6 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, has_base = ir1_opnd_has_base(opnd1); #ifdef CONFIG_LATX_IMM_REG - if (!option_imm_reg || !option_imm_complex) { - cache_skip = true; - } int base = -1; int index = -1; int scale = ir1_opnd_scale(opnd1); @@ -339,13 +369,50 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, scale = -1; } + complex_kind = imm_cache_complex_kind(has_base, has_index, offset); + if (complex_kind < 0) { + cache_skip = true; + imm_cache->complex_stats + .skips[IMM_CACHE_COMPLEX_SKIP_UNSUPPORTED_FORM]++; + } else if (!option_imm_reg || + !(option_imm_complex & (1 << complex_kind))) { + cache_skip = true; + imm_cache->complex_stats.skips[IMM_CACHE_COMPLEX_SKIP_DISABLED]++; +#ifndef TARGET_X86_64 + } else if (complex_kind >= 0) { + cache_skip = true; + imm_cache->complex_stats.skips[IMM_CACHE_COMPLEX_SKIP_NOT_CODE64]++; +#else + } else if (!CODEIS64) { + cache_skip = true; + imm_cache->complex_stats.skips[IMM_CACHE_COMPLEX_SKIP_NOT_CODE64]++; + } else if (addr_size != 64) { + cache_skip = true; + imm_cache->complex_stats.skips[IMM_CACHE_COMPLEX_SKIP_ADDR_SIZE]++; + } else if (has_seg) { + cache_skip = true; + imm_cache->complex_stats.skips[IMM_CACHE_COMPLEX_SKIP_SEGMENT]++; + } else if (has_index && scale < 0) { + cache_skip = true; + imm_cache->complex_stats.skips[IMM_CACHE_COMPLEX_SKIP_INVALID_SCALE]++; +#endif + } else if (cache_skip) { + imm_cache->complex_stats.skips[IMM_CACHE_COMPLEX_SKIP_INSTRUCTION]++; + } + #endif if (host_off) { if (si12_overflow(offset)) { *host_off = 0; } else { #ifdef CONFIG_LATX_IMM_REG - cache_skip = true; + if (!cache_skip && + complex_kind == IMM_CACHE_COMPLEX_BASE_DISP) { + cache_skip = true; + imm_cache->complex_stats + .skips[IMM_CACHE_COMPLEX_SKIP_NO_BENEFIT]++; + } + complex_host_off = offset; #endif /* ifdef CONFIG_LATX_IMM_REG */ *host_off = offset; @@ -365,6 +432,10 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, #ifdef CONFIG_LATX_IMM_REG // i need base, index, scale offset if (!dest_need_itmp) { + if (!cache_skip && complex_kind >= 0) { + imm_cache->complex_stats + .skips[IMM_CACHE_COMPLEX_SKIP_FIXED_DEST]++; + } goto imm_cache_exit; } if (cache_skip) { @@ -380,6 +451,7 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, */ IMM_CACHE_RES res = imm_cache_allocate(imm_cache, base, index, scale, offset); + complex_attempted = true; dest_op = ra_alloc_imm_reg(res.itemp_num); imm_log("[complex]bitmap:%d\n", bitmap); // check if alloc reg conflict with itemp alloc @@ -389,6 +461,7 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, dest_op = ra_alloc_itemp(); free_imm_reg(res.itemp_num); imm_cache->itemp_allocated = true; + imm_cache->complex_stats.itemp_conflicts[complex_kind]++; goto imm_cache_exit; } // reg allocate success. @@ -446,17 +519,35 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, } //? if (host_off) { - *host_off = res.diff; + long host_diff = complex_host_off + res.diff; + + if (!si12_overflow(host_diff)) { + *host_off = host_diff; + imm_cache->complex_stats + .host_offset_hits[complex_kind]++; + } else { + imm_cache_update_by_diff(imm_cache, res.cache_id, + res.diff); + la_addi_d(dest_op, dest_op, res.diff); + *host_off = complex_host_off; + imm_cache->complex_stats.addi_hits[complex_kind]++; + } offset = 0; } else if (res.diff != 0) { imm_cache_update_by_diff(imm_cache, res.cache_id, res.diff); la_addi_d(dest_op, dest_op, res.diff); + imm_cache->complex_stats.addi_hits[complex_kind]++; + } else { + imm_cache->complex_stats.direct_hits[complex_kind]++; } goto skip; } imm_cache_exit: + if (complex_attempted) { + imm_cache->complex_stats.full_generations[complex_kind]++; + } #endif switch (bitmap) { @@ -554,7 +645,6 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, lsassert(arg_dest_op); } - addr_size = ir1_addr_size(pir1); return adjust_dest(dest_op, arg_dest_op, dest_size, addr_size); } diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 38266d7bbcb..a51b52b0e9e 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -135,7 +135,7 @@ void tr_init(void *tb) } if (t->imm_cache == NULL) { - t->imm_cache = (IMM_CACHE *)mm_malloc(sizeof(IMM_CACHE)); + t->imm_cache = (IMM_CACHE *)mm_calloc(1, sizeof(IMM_CACHE)); t->imm_cache->bucket = (IMM_CACHE_BUCKET *)mm_calloc( CACHE_MAX_CAPACITY, sizeof(IMM_CACHE_BUCKET)); } @@ -2360,16 +2360,13 @@ int tr_ir2_generate(struct TranslationBlock *tb) continue; } #endif - /* convert_mem_helper will put si12 offset into host_off */ - if (!si12_overflow(offset)) { - continue; - } // record base and index bool has_index = ir1_opnd_has_index(opnd); bool has_base = ir1_opnd_has_base(opnd); int base_op = -1; int index_op = -1; int scale = -1; + int complex_kind = -1; if (has_base) { base_op = ir1_opnd_base_reg_num(opnd); } @@ -2382,6 +2379,38 @@ int tr_ir2_generate(struct TranslationBlock *tb) } } + if (has_base && has_index) { + complex_kind = IMM_CACHE_COMPLEX_BASE_INDEX_DISP; + } else if (has_index) { + complex_kind = IMM_CACHE_COMPLEX_INDEX_DISP; + } else if (has_base && offset) { + complex_kind = IMM_CACHE_COMPLEX_BASE_DISP; + } + + if (complex_kind < 0 || + !(option_imm_complex & (1 << complex_kind)) || +#ifdef TARGET_X86_64 + !CODEIS64 || +#endif + ir1_addr_size(t_pir1) != 64 || + ir1_opnd_has_seg(opnd) || + (has_index && scale < 0)) { + continue; + } + + /* + * convert_mem_helper leaves a small displacement in the + * host load/store. Only base+disp has no address cache + * work left in that case. The other modes cache the + * base/index expression at offset zero. + */ + if (!si12_overflow(offset)) { + if (complex_kind == IMM_CACHE_COMPLEX_BASE_DISP) { + continue; + } + offset = 0; + } + if (base_op != -1 || index_op != -1) { // qemu_log_mask(LAT_IMM_REG,"[imm_cache-]") imm_cache_precache_put(imm_cache, base_op, @@ -2455,6 +2484,12 @@ int tr_ir2_generate(struct TranslationBlock *tb) pir1++; } +#ifdef CONFIG_LATX_IMM_REG + if (option_imm_reg) { + imm_cache_print_rip_stats(imm_cache, tb->pc); + imm_cache_print_complex_stats(imm_cache, tb->pc); + } +#endif #ifdef CONFIG_LATX_DEBUG if (option_dump_ir1) { pir1 = tb_ir1_inst(tb, 0); @@ -4140,6 +4175,9 @@ void tr_load_x64_8_registers_from_env(uint8 gpr_to_load, uint8 xmm_to_load) void tr_gen_call_to_helper(ADDR func_addr, enum aot_rel_kind REL_KIND) { +#ifdef CONFIG_LATX_IMM_REG + imm_cache_invalidate_for_helper(); +#endif IR2_OPND func_addr_opnd = ra_alloc_dbt_arg2(); TranslationBlock *tb __attribute__((unused)) = NULL; if (option_aot) { @@ -4178,6 +4216,9 @@ void convert_fpregs_x80_to_64(void) static void tr_gen_call_to_helper_prologue(int use_fp) { +#ifdef CONFIG_LATX_IMM_REG + imm_cache_invalidate_for_helper(); +#endif tr_save_registers_to_env(0, FPR_USEDEF_TO_SAVE, XMM_USEDEF_TO_SAVE, options_to_save()); #ifdef TARGET_X86_64 diff --git a/tests/integration/complex-imm-cache.S b/tests/integration/complex-imm-cache.S new file mode 100644 index 00000000000..fa533c0a245 --- /dev/null +++ b/tests/integration/complex-imm-cache.S @@ -0,0 +1,684 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + + .equ __NR_rt_sigaction, 13 + .equ __NR_rt_sigreturn, 15 + .equ __NR_mmap, 9 + .equ __NR_munmap, 11 + .equ __NR_getpid, 39 + .equ __NR_fork, 57 + .equ __NR_exit, 60 + .equ __NR_wait4, 61 + .equ __NR_kill, 62 + .equ __NR_arch_prctl, 158 + .equ ARCH_SET_GS, 0x1001 + .equ PROT_RWX, 7 + .equ MAP_PRIVATE_ANONYMOUS, 0x22 + .equ SIGUSR1, 10 + .equ SA_RESTORER, 0x04000000 + + .section .text + .globl _start + .type _start, @function +_start: + call .Ltest_base_disp + test %eax, %eax + jne .Lexit + call .Ltest_index_disp + test %eax, %eax + jne .Lexit + call .Ltest_base_index_disp + test %eax, %eax + jne .Lexit + call .Ltest_register_writes + test %eax, %eax + jne .Lexit + call .Ltest_enter_leave + test %eax, %eax + jne .Lexit + call .Ltest_string_writes + test %eax, %eax + jne .Lexit + call .Ltest_xlat_write + test %eax, %eax + jne .Lexit + call .Ltest_precache_mixed_access + test %eax, %eax + jne .Lexit + call .Ltest_cache_pressure + test %eax, %eax + jne .Lexit + call .Ltest_helper + test %eax, %eax + jne .Lexit + call .Ltest_excluded_addressing + test %eax, %eax + jne .Lexit + call .Ltest_signal + test %eax, %eax + jne .Lexit + call .Ltest_smc + test %eax, %eax + jne .Lexit + call .Ltest_fork + +.Lexit: + mov %eax, %edi + mov $__NR_exit, %eax + syscall + .size _start, .-_start + +.Ltest_base_disp: + lea base_data(%rip), %r8 + mov 4096(%r8), %eax + mov 4104(%r8), %ecx + cmp $0x11, %eax + jne .Lfail + cmp $0x22, %ecx + jne .Lfail + + lea base_anchor(%rip), %r8 + mov -4096(%r8), %eax + mov -4088(%r8), %ecx + cmp $0x11, %eax + jne .Lfail + cmp $0x22, %ecx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_index_disp: + xor %r9d, %r9d + mov index_data(,%r9,1), %eax + mov index_data + 8(,%r9,1), %ecx + cmp $0x31, %eax + jne .Lfail + cmp $0x32, %ecx + jne .Lfail + + mov index_data(,%r9,2), %eax + mov index_data + 8(,%r9,2), %ecx + cmp $0x31, %eax + jne .Lfail + cmp $0x32, %ecx + jne .Lfail + + mov index_data(,%r9,4), %eax + mov index_data + 8(,%r9,4), %ecx + cmp $0x31, %eax + jne .Lfail + cmp $0x32, %ecx + jne .Lfail + + mov index_data(,%r9,8), %eax + mov index_data + 8(,%r9,8), %ecx + cmp $0x31, %eax + jne .Lfail + cmp $0x32, %ecx + jne .Lfail + + lea index_data + 4096(%rip), %r9 + mov -4096(,%r9,1), %eax + mov -4088(,%r9,1), %ecx + cmp $0x31, %eax + jne .Lfail + cmp $0x32, %ecx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_base_index_disp: + lea combined_data(%rip), %r8 + xor %r9d, %r9d + mov 0(%r8,%r9,1), %eax + mov 8(%r8,%r9,1), %ecx + cmp $0x41, %eax + jne .Lfail + cmp $0x42, %ecx + jne .Lfail + + mov 0(%r8,%r9,2), %eax + mov 8(%r8,%r9,2), %ecx + cmp $0x41, %eax + jne .Lfail + cmp $0x42, %ecx + jne .Lfail + + mov 0(%r8,%r9,4), %eax + mov 8(%r8,%r9,4), %ecx + cmp $0x41, %eax + jne .Lfail + cmp $0x42, %ecx + jne .Lfail + + mov 0(%r8,%r9,8), %eax + mov 8(%r8,%r9,8), %ecx + cmp $0x41, %eax + jne .Lfail + cmp $0x42, %ecx + jne .Lfail + + lea combined_data + 4096(%rip), %r8 + xor %r9d, %r9d + mov -4096(%r8,%r9,1), %eax + mov -4088(%r8,%r9,1), %ecx + cmp $0x41, %eax + jne .Lfail + cmp $0x42, %ecx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_register_writes: + /* XCHG writes both explicit operands. */ + lea write_data(%rip), %r8 + lea write_data + 8(%rip), %r10 + xor %r9d, %r9d + mov (%r8,%r9,1), %eax + xchg %r8, %r10 + mov (%r8,%r9,1), %ecx + cmp $0x51, %eax + jne .Lfail + cmp $0x52, %ecx + jne .Lfail + + lea write_data(%rip), %r8 + xor %r9d, %r9d + mov $8, %r10 + mov (%r8,%r9,1), %eax + xchg %r9, %r10 + mov (%r8,%r9,1), %ecx + cmp $0x51, %eax + jne .Lfail + cmp $0x52, %ecx + jne .Lfail + + /* XADD also writes its source operand. */ + lea write_data(%rip), %r8 + xor %r9d, %r9d + mov $8, %r10 + mov (%r8,%r9,1), %eax + xadd %r9, %r10 + mov (%r8,%r9,1), %ecx + cmp $0x51, %eax + jne .Lfail + cmp $0x52, %ecx + jne .Lfail + + /* MULX writes both explicit destinations. */ + lea write_data(%rip), %r8 + xor %r9d, %r9d + mov (%r8,%r9,1), %eax + lea write_data + 8(%rip), %rdx + mov $1, %r10 + mulx %r10, %r8, %r11 + mov (%r8,%r9,1), %ecx + cmp $0x51, %eax + jne .Lfail + cmp $0x52, %ecx + jne .Lfail + + /* A 32-bit destination write zero-extends and invalidates the base. */ + lea write_data(%rip), %r8 + mov (%r8,%r9,1), %eax + mov $write_data + 8, %r8d + mov (%r8,%r9,1), %ecx + cmp $0x51, %eax + jne .Lfail + cmp $0x52, %ecx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_enter_leave: + sub $128, %rsp + mov %rbp, %r11 + lea -64(%rsp), %rbp + mov %ebp, %r10d + movl $0xa1, 0(%rsp) + movl $0xa2, -24(%rsp) + movl $0xa3, 0(%rbp) + xor %r9d, %r9d + + /* Cache addresses based on the pre-ENTER RSP and RBP. */ + mov 0(%rsp,%r9,1), %eax + mov 0(%rbp,%r9,1), %ebx + enter $16, $0 + + /* ENTER changes both RSP and RBP. */ + mov 0(%rsp,%r9,1), %ecx + mov 0(%rbp,%r9,1), %edx + leave + + /* LEAVE changes both registers again. */ + mov 0(%rsp,%r9,1), %esi + mov 0(%rbp,%r9,1), %edi + mov %r11, %rbp + add $128, %rsp + + cmp $0xa1, %eax + jne .Lfail + cmp $0xa3, %ebx + jne .Lfail + cmp $0xa2, %ecx + jne .Lfail + cmp %r10d, %edx + jne .Lfail + cmp $0xa1, %esi + jne .Lfail + cmp $0xa3, %edi + jne .Lfail + xor %eax, %eax + ret + +.Ltest_string_writes: + cld + xor %r9d, %r9d + + /* LODS advances RSI. */ + lea string_lods_data(%rip), %rsi + movzbl (%rsi,%r9,1), %r10d + lodsb + movzbl (%rsi,%r9,1), %ecx + cmp $0xb1, %r10d + jne .Lfail + cmp $0xb2, %ecx + jne .Lfail + + /* STOS and SCAS advance RDI. */ + lea string_stos_data(%rip), %rdi + movzbl (%rdi,%r9,1), %r10d + mov $0xbf, %al + stosb + movzbl (%rdi,%r9,1), %ecx + cmp $0xc1, %r10d + jne .Lfail + cmp $0xc2, %ecx + jne .Lfail + + lea string_scas_data(%rip), %rdi + movzbl (%rdi,%r9,1), %r10d + mov $0xd1, %al + scasb + movzbl (%rdi,%r9,1), %ecx + cmp $0xd1, %r10d + jne .Lfail + cmp $0xd2, %ecx + jne .Lfail + + /* MOVS and CMPS advance both RSI and RDI. */ + lea string_movs_src(%rip), %rsi + lea string_movs_dst(%rip), %rdi + movzbl (%rsi,%r9,1), %r10d + movzbl (%rdi,%r9,1), %r11d + movsb + movzbl (%rsi,%r9,1), %ecx + movzbl (%rdi,%r9,1), %edx + cmp $0xe1, %r10d + jne .Lfail + cmp $0xf1, %r11d + jne .Lfail + cmp $0xe2, %ecx + jne .Lfail + cmp $0xf2, %edx + jne .Lfail + + lea string_cmps_src(%rip), %rsi + lea string_cmps_dst(%rip), %rdi + movzbl (%rsi,%r9,1), %r10d + movzbl (%rdi,%r9,1), %r11d + cmpsb + movzbl (%rsi,%r9,1), %ecx + movzbl (%rdi,%r9,1), %edx + cmp $0x21, %r10d + jne .Lfail + cmp $0x31, %r11d + jne .Lfail + cmp $0x22, %ecx + jne .Lfail + cmp $0x32, %edx + jne .Lfail + + /* REP string operations also consume RCX. */ + lea string_rep_addr(%rip), %r8 + mov $1, %ecx + movzbl (%r8,%rcx,1), %r10d + lea string_rep_src(%rip), %rsi + lea string_rep_dst(%rip), %rdi + rep movsb + movzbl (%r8,%rcx,1), %edx + cmp $0x42, %r10d + jne .Lfail + cmp $0x41, %edx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_xlat_write: + lea xlat_target(%rip), %rax + lea xlat_table(%rip), %rbx + xor %r9d, %r9d + mov (%rax,%r9,1), %r10d + xlatb + mov (%rax,%r9,1), %ecx + cmp $0x71, %r10d + jne .Lfail + cmp $0x72, %ecx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_precache_mixed_access: + lea precache_mixed_data(%rip), %r8 + xor %eax, %eax + xor %edx, %edx + mov $0x11223344, %ebx + mov $0x55667788, %ecx + cmpxchg8b 4097(%r8) + mov 4096(%r8), %eax + mov 4104(%r8), %ecx + cmp $0x44332211, %eax + jne .Lfail + cmp $0x88776655, %ecx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_cache_pressure: + lea pressure_data(%rip), %r8 + lea pressure_data(%rip), %r9 + lea pressure_data(%rip), %r10 + lea pressure_data(%rip), %r11 + lea pressure_data(%rip), %r12 + xor %r15d, %r15d + + mov 0(%r8,%r15,1), %eax + mov 8(%r8,%r15,1), %ecx + mov 0(%r9,%r15,1), %edx + mov 8(%r9,%r15,1), %esi + mov 0(%r10,%r15,1), %edi + mov 8(%r10,%r15,1), %ebp + mov 0(%r11,%r15,1), %ebx + mov 8(%r11,%r15,1), %r14d + mov 0(%r12,%r15,1), %r13d + mov 8(%r12,%r15,1), %r12d + + cmp $0x91, %eax + jne .Lfail + cmp $0x92, %ecx + jne .Lfail + cmp $0x91, %edx + jne .Lfail + cmp $0x92, %esi + jne .Lfail + cmp $0x91, %edi + jne .Lfail + cmp $0x92, %ebp + jne .Lfail + cmp $0x91, %ebx + jne .Lfail + cmp $0x92, %r14d + jne .Lfail + cmp $0x91, %r13d + jne .Lfail + cmp $0x92, %r12d + jne .Lfail + xor %eax, %eax + ret + +.Ltest_helper: + lea helper_data(%rip), %r8 + xor %r9d, %r9d + mov (%r8,%r9,1), %eax + pxor %xmm0, %xmm0 + pcmpistri $0, %xmm0, %xmm0 + fld1 + fsin + fstp %st(0) + mov 8(%r8,%r9,1), %ecx + cmp $0x61, %eax + jne .Lfail + cmp $0x62, %ecx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_excluded_addressing: + xor %esi, %esi + mov $ARCH_SET_GS, %edi + mov $__NR_arch_prctl, %eax + syscall + test %eax, %eax + js .Lfail + + lea excluded_data(%rip), %r8 + xor %r9d, %r9d + mov %gs:(%r8,%r9,1), %eax + mov %gs:8(%r8,%r9,1), %ecx + cmp $0x71, %eax + jne .Lfail + cmp $0x72, %ecx + jne .Lfail + + mov $excluded_data, %r8d + xor %r9d, %r9d + addr32 mov (%r8d,%r9d,1), %eax + addr32 mov 8(%r8d,%r9d,1), %ecx + cmp $0x71, %eax + jne .Lfail + cmp $0x72, %ecx + jne .Lfail + xor %eax, %eax + ret + +.Ltest_signal: + lea .Lsignal_handler(%rip), %rax + mov %rax, signal_action(%rip) + lea .Lsignal_restorer(%rip), %rax + mov %rax, signal_action + 16(%rip) + mov $__NR_rt_sigaction, %eax + mov $SIGUSR1, %edi + lea signal_action(%rip), %rsi + xor %edx, %edx + mov $8, %r10d + syscall + test %eax, %eax + js .Lfail + + mov $__NR_getpid, %eax + syscall + mov %eax, %edi + mov $SIGUSR1, %esi + mov $__NR_kill, %eax + syscall + test %eax, %eax + js .Lfail + cmpb $1, signal_seen(%rip) + jne .Lfail + xor %eax, %eax + ret + +.Lsignal_handler: + lea signal_data(%rip), %r8 + xor %r9d, %r9d + mov (%r8,%r9,1), %eax + mov 8(%r8,%r9,1), %ecx + cmp $0x81, %eax + jne .Lsignal_handler_fail + cmp $0x82, %ecx + jne .Lsignal_handler_fail + movb $1, signal_seen(%rip) + ret + +.Lsignal_handler_fail: + movb $42, signal_seen(%rip) + ret + +.Lsignal_restorer: + mov $__NR_rt_sigreturn, %eax + syscall + +.Ltest_smc: + mov $__NR_mmap, %eax + xor %edi, %edi + mov $4096, %esi + mov $PROT_RWX, %edx + mov $MAP_PRIVATE_ANONYMOUS, %r10d + mov $-1, %r8d + xor %r9d, %r9d + syscall + test %rax, %rax + js .Lfail + mov %rax, %r13 + + /* mov (%rdi,%rsi),eax; mov 4(%rdi,%rsi),ecx; add ecx,eax; ret */ + movabs $0x0104374c8b37048b, %rax + mov %rax, 0(%r13) + movw $0xc3c8, 8(%r13) + lea smc_data(%rip), %rdi + xor %esi, %esi + call *%r13 + cmp $3, %eax + jne .Lsmc_unmap_fail + + movb $8, 6(%r13) + lea smc_data(%rip), %rdi + xor %esi, %esi + call *%r13 + mov %eax, %r12d + + mov $__NR_munmap, %eax + mov %r13, %rdi + mov $4096, %esi + syscall + test %eax, %eax + js .Lfail + cmp $5, %r12d + jne .Lfail + xor %eax, %eax + ret + +.Lsmc_unmap_fail: + mov $__NR_munmap, %eax + mov %r13, %rdi + mov $4096, %esi + syscall + jmp .Lfail + +.Ltest_fork: + mov $__NR_fork, %eax + syscall + test %eax, %eax + js .Lfail + jz .Lfork_child + + mov %eax, %edi + lea child_status(%rip), %rsi + xor %edx, %edx + xor %r10d, %r10d + mov $__NR_wait4, %eax + syscall + test %eax, %eax + js .Lfail + cmpl $0, child_status(%rip) + jne .Lfail + xor %eax, %eax + ret + +.Lfork_child: + call .Ltest_base_index_disp + mov %eax, %edi + mov $__NR_exit, %eax + syscall + +.Lfail: + mov $42, %eax + ret + + .section .data + .balign 16 +base_data: + .space 4096 + .quad 0x11, 0x22 + .space 4080 +base_anchor: + + .balign 16 +index_data: + .quad 0x31, 0x32 + .space 4080 + + .balign 16 +combined_data: + .quad 0x41, 0x42 + .space 4080 + + .balign 16 +write_data: + .quad 0x51, 0x52 + + .balign 16 +helper_data: + .quad 0x61, 0x62 + + .balign 16 +excluded_data: + .quad 0x71, 0x72 + + .balign 16 +signal_data: + .quad 0x81, 0x82 + + .balign 16 +smc_data: + .long 1, 2, 4 + + .balign 16 +pressure_data: + .quad 0x91, 0x92 + +string_lods_data: + .byte 0xb1, 0xb2 +string_stos_data: + .byte 0xc1, 0xc2 +string_scas_data: + .byte 0xd1, 0xd2 +string_movs_src: + .byte 0xe1, 0xe2 +string_movs_dst: + .byte 0xf1, 0xf2 +string_cmps_src: + .byte 0x21, 0x22 +string_cmps_dst: + .byte 0x31, 0x32 +string_rep_addr: + .byte 0x41, 0x42 +string_rep_src: + .byte 0x51 +string_rep_dst: + .byte 0x61 + + .balign 256 +xlat_target: + .long 0x71 + .long 0 + .long 0x72 +xlat_table: + .byte 8 + + .balign 16 +precache_mixed_data: + .space 4096 + .byte 0x11, 0x22, 0x33, 0x44 + .long 0 + .byte 0x55, 0x66, 0x77, 0x88 + + .balign 8 +signal_action: + .quad 0 + .quad SA_RESTORER + .quad 0 + .quad 0 +child_status: + .long 0 +signal_seen: + .byte 0 diff --git a/tests/integration/registrations/process/meson.build b/tests/integration/registrations/process/meson.build index bf2f8406bca..2635da07e7a 100644 --- a/tests/integration/registrations/process/meson.build +++ b/tests/integration/registrations/process/meson.build @@ -73,6 +73,22 @@ if 'x86_64-linux-user' in target_dirs ], }] endif + latx_integration_tests += [{ + 'name': 'test-rip-imm-cache', + 'runner': find_program('../../test-rip-imm-cache.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../rip-imm-cache.S'), + ], + }] + latx_integration_tests += [{ + 'name': 'test-complex-imm-cache', + 'runner': find_program('../../test-complex-imm-cache.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../complex-imm-cache.S'), + ], + }] if 'CONFIG_LATX_KZT' in config_host latx_integration_tests += [{ 'name': 'test-int3-sigtrap-rip', diff --git a/tests/integration/rip-imm-cache.S b/tests/integration/rip-imm-cache.S new file mode 100644 index 00000000000..d27b39e38d8 --- /dev/null +++ b/tests/integration/rip-imm-cache.S @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + + .equ __NR_rt_sigaction, 13 + .equ __NR_rt_sigreturn, 15 + .equ __NR_mmap, 9 + .equ __NR_munmap, 11 + .equ __NR_getpid, 39 + .equ __NR_fork, 57 + .equ __NR_execve, 59 + .equ __NR_exit, 60 + .equ __NR_wait4, 61 + .equ __NR_kill, 62 + .equ PROT_RWX, 7 + .equ MAP_PRIVATE_ANONYMOUS, 0x22 + .equ SIGUSR1, 10 + .equ SA_RESTORER, 0x04000000 + + .section .text + .balign 8 +negative_data: + .quad 0x11 + + .globl _start + .type _start, @function +_start: + cmpq $1, (%rsp) + jne .Lcore_only + mov 8(%rsp), %rax + mov %rax, saved_argv0(%rip) + movb $1, run_lifecycle(%rip) + +.Lcore_only: + call .Lrun_core + test %eax, %eax + jne .Lexit + + call .Ltest_signal + test %eax, %eax + jne .Lexit + + cmpb $0, run_lifecycle(%rip) + je .Lexit + + mov $__NR_fork, %eax + syscall + test %eax, %eax + js .Lfail + jz .Lfork_child + + mov %eax, %edi + lea child_status(%rip), %rsi + xor %edx, %edx + xor %r10d, %r10d + mov $__NR_wait4, %eax + syscall + test %eax, %eax + js .Lfail + cmpl $0, child_status(%rip) + jne .Lfail + + mov saved_argv0(%rip), %rdi + mov %rdi, exec_argv(%rip) + lea exec_marker(%rip), %rax + mov %rax, exec_argv + 8(%rip) + lea exec_argv(%rip), %rsi + xor %edx, %edx + mov $__NR_execve, %eax + syscall + jmp .Lfail + +.Lfork_child: + call .Lrun_core + mov %eax, %edi + mov $__NR_exit, %eax + syscall + +.Lrun_core: + mov negative_data(%rip), %r8 + mov positive_data(%rip), %r9 + add positive_data(%rip), %r9 + xor distant_data(%rip), %r8 + add %r9, %r8 + + /* Keep GOTPCREL references unrelaxed in the PIE/shared variants. */ + lea positive_data@GOTPCREL(%rip), %r11 + lea positive_data@GOTPCREL(%rip), %r12 + cmp %r11, %r12 + jne .Lcore_fail + + /* Exercise both common and raw helper-call paths with a live RIP cache. */ + pxor %xmm0, %xmm0 + pcmpistri $0, %xmm0, %xmm0 + fld1 + fsin + fstp %st(0) + mov positive_data(%rip), %r10 + cmp $0x22, %r10 + jne .Lcore_fail + mov %fs:positive_data(%rip), %r10 + cmp $0x22, %r10 + jne .Lcore_fail + cmp $0x99, %r8 + jne .Lcore_fail + + call .Lcross_page + test %eax, %eax + jne .Lcore_fail + + call .Ltest_smc + test %eax, %eax + jne .Lcore_fail + + xor %eax, %eax + ret + +.Lcore_fail: + mov $42, %eax + ret + +.Ltest_signal: + lea .Lsignal_handler(%rip), %rax + mov %rax, signal_action(%rip) + lea .Lsignal_restorer(%rip), %rax + mov %rax, signal_action + 16(%rip) + mov $__NR_rt_sigaction, %eax + mov $SIGUSR1, %edi + lea signal_action(%rip), %rsi + xor %edx, %edx + mov $8, %r10d + syscall + test %eax, %eax + js .Lsignal_fail + + mov $__NR_getpid, %eax + syscall + mov %eax, %edi + mov $SIGUSR1, %esi + mov $__NR_kill, %eax + syscall + test %eax, %eax + js .Lsignal_fail + cmpb $1, signal_seen(%rip) + jne .Lsignal_fail + + xor %eax, %eax + ret + +.Lsignal_fail: + mov $42, %eax + ret + +.Lsignal_handler: + mov positive_data(%rip), %rax + cmp positive_data(%rip), %rax + jne .Lsignal_handler_fail + movb $1, signal_seen(%rip) + ret + +.Lsignal_handler_fail: + movb $42, signal_seen(%rip) + ret + +.Lsignal_restorer: + mov $__NR_rt_sigreturn, %eax + syscall + +.Ltest_smc: + mov $__NR_mmap, %eax + xor %edi, %edi + mov $4096, %esi + mov $PROT_RWX, %edx + mov $MAP_PRIVATE_ANONYMOUS, %r10d + mov $-1, %r8d + xor %r9d, %r9d + syscall + test %rax, %rax + js .Lsmc_fail + mov %rax, %r13 + + movabs $0x0d8b0000000a058b, %rax + mov %rax, 0(%r13) + movabs $0x90c3c80100000004, %rax + mov %rax, 8(%r13) + movabs $0x0000002200000011, %rax + mov %rax, 16(%r13) + call *%r13 + cmp $0x22, %eax + jne .Lsmc_unmap_fail + + /* Retarget both RIP-relative loads from data[0] to data[1]. */ + movb $0x0e, 2(%r13) + movb $0x08, 8(%r13) + call *%r13 + mov %eax, %r12d + + mov $__NR_munmap, %eax + mov %r13, %rdi + mov $4096, %esi + syscall + test %eax, %eax + js .Lsmc_fail + cmp $0x44, %r12d + jne .Lsmc_fail + xor %eax, %eax + ret + +.Lsmc_unmap_fail: + mov $__NR_munmap, %eax + mov %r13, %rdi + mov $4096, %esi + syscall +.Lsmc_fail: + mov $42, %eax + ret + +.Lfail: + mov $42, %eax +.Lexit: + mov %eax, %edi + mov $__NR_exit, %eax + syscall + .size _start, .-_start + + /* The first load straddles a 4 KiB guest page boundary. */ + .balign 4096 + .space 4092, 0x90 +.Lcross_page: + mov positive_data(%rip), %eax + cmp $0x22, %eax + jne .Lcross_page_fail + mov positive_data(%rip), %ecx + cmp $0x22, %ecx + jne .Lcross_page_fail + xor %eax, %eax + ret + +.Lcross_page_fail: + mov $42, %eax + ret + + .section .data + .balign 8 +positive_data: + .quad 0x22 + .space 4096 +distant_data: + .quad 0x44 + + .balign 8 +signal_action: + .quad 0 + .quad SA_RESTORER + .quad 0 + .quad 0 +saved_argv0: + .quad 0 +exec_argv: + .quad 0, 0, 0 +child_status: + .long 0 +signal_seen: + .byte 0 +run_lifecycle: + .byte 0 +exec_marker: + .asciz "exec" + + .section .note.GNU-stack,"",@progbits diff --git a/tests/integration/test-complex-imm-cache.sh b/tests/integration/test-complex-imm-cache.sh new file mode 100755 index 00000000000..156276ebadb --- /dev/null +++ b/tests/integration/test-complex-imm-cache.sh @@ -0,0 +1,117 @@ +#!/bin/sh +set -eu + +emulator=$(readlink -f "$1") +source_file=$(readlink -f "$2") +tmp_root=${TMPDIR:-${HOME}/tmp} +mkdir -p "$tmp_root" +workdir=$(mktemp -d "$tmp_root/complex-imm-cache.XXXXXX") +cleanup() +{ + for attempt in 1 2 3 4 5; do + if rm -rf "$workdir"; then + return + fi + sleep 0.1 + done + return 1 +} +trap cleanup EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +"$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none -Wl,--no-relax "$source_file" \ + -o "$workdir/complex-static" + +run_guest() +{ + timeout 10s env LATX_AOT=0 LATX_KZT=0 LATX_VPAES=0 \ + "$@" "$emulator" "$workdir/complex-static" +} + +run_guest LATX_IMM_COMPLEX=0 +run_guest LATX_IMM_REG=1111 LATX_IMM_COMPLEX=7 +run_guest LATX_IMM_COMPLEX_STATS=1 2>"$workdir/default.err" + +for mask in 1 2 4 7; do + run_guest LATX_IMM_COMPLEX="$mask" LATX_IMM_COMPLEX_STATS=1 \ + 2>"$workdir/mask-$mask.err" +done + +mkdir -p "$workdir/aot-home" +for pass in 1 2; do + timeout 10s env HOME="$workdir/aot-home" LATX_AOT=1 LATX_KZT=0 \ + LATX_VPAES=0 LATX_IMM_COMPLEX=7 \ + "$emulator" "$workdir/complex-static" +done + +if run_guest LATX_IMM_COMPLEX=invalid >/dev/null 2>&1; then + echo "FAIL: invalid LATX_IMM_COMPLEX was accepted" >&2 + exit 1 +fi +if run_guest LATX_IMM_COMPLEX=8 >/dev/null 2>&1; then + echo "FAIL: out-of-range LATX_IMM_COMPLEX was accepted" >&2 + exit 1 +fi +if run_guest LATX_IMM_COMPLEX_STATS=2 >/dev/null 2>&1; then + echo "FAIL: invalid LATX_IMM_COMPLEX_STATS was accepted" >&2 + exit 1 +fi + +run_guest LATX_IMM_COMPLEX=0 LATX_IMM_COMPLEX_STATS=1 \ + 2>"$workdir/disabled.err" +if grep -Eq '^\[LATX\]\[imm-complex\].*calls=[1-9][0-9]*' \ + "$workdir/disabled.err"; then + echo "FAIL: disabled complex cache attempted a mode" >&2 + exit 1 +fi + +check_mode() +{ + mask=$1 + mode=$2 + grep -E "^\\[LATX\\]\\[imm-complex\\].*mode=$mode" \ + "$workdir/mask-$mask.err" | + grep -Eq 'calls=[1-9][0-9]*.*hits=[1-9][0-9]*' +} + +check_mode 1 base-disp +check_mode 2 index-disp +check_mode 4 base-index-disp +check_mode 7 base-disp +check_mode 7 index-disp +check_mode 7 base-index-disp + +for mode in base-disp index-disp base-index-disp; do + grep -E "^\[LATX\]\[imm-complex\].*mode=$mode" \ + "$workdir/default.err" | + grep -Eq 'calls=[1-9][0-9]*.*hits=[1-9][0-9]*' +done + +grep -Eq '^\[LATX\]\[imm-complex-skip\].*addr-size=[1-9][0-9]*' \ + "$workdir/mask-7.err" +grep -Eq '^\[LATX\]\[imm-complex-skip\].*segment=[1-9][0-9]*' \ + "$workdir/mask-7.err" +grep -Eq '^\[LATX\]\[imm-complex-skip\].*base-index-invalidations=[1-9][0-9]*' \ + "$workdir/mask-7.err" +grep -E '^\[LATX\]\[imm-complex\].*mode=base-index-disp' \ + "$workdir/mask-7.err" | + grep -Eq 'replacements=[1-9][0-9]*' +LATX_IMM_COMPLEX=7 LATX_IMM_COMPLEX_STATS=1 \ + timeout 10s "$emulator" -latx-imm-complex 0 \ + "$workdir/complex-static" 2>"$workdir/override.err" +if grep -Eq '^\[LATX\]\[imm-complex\].*calls=[1-9][0-9]*' \ + "$workdir/override.err"; then + echo "FAIL: command-line disable did not override the environment" >&2 + exit 1 +fi + +echo "PASS: complex immediate cache modes, JIT/AOT, and safety matrix" diff --git a/tests/integration/test-rip-imm-cache.sh b/tests/integration/test-rip-imm-cache.sh new file mode 100755 index 00000000000..b8f74492ffe --- /dev/null +++ b/tests/integration/test-rip-imm-cache.sh @@ -0,0 +1,75 @@ +#!/bin/sh +set -eu + +emulator=$(readlink -f "$1") +source_file=$(readlink -f "$2") +tmp_root=${TMPDIR:-${HOME}/tmp} +mkdir -p "$tmp_root" +workdir=$(mktemp -d "$tmp_root/rip-imm-cache.XXXXXX") +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +common_flags="--target=x86_64-linux-gnu -fuse-ld=lld -nostdlib" +linker_flags="-Wl,--build-id=none -Wl,--no-relax" +"$clang" $common_flags -static $linker_flags \ + "$source_file" -o "$workdir/rip-static" +"$clang" $common_flags -static-pie $linker_flags \ + "$source_file" -o "$workdir/rip-pie" +"$clang" $common_flags -shared -Wl,-e,_start $linker_flags \ + "$source_file" -o "$workdir/rip-shared" + +run_guest() +{ + guest=$1 + shift + timeout 10s env LATX_AOT=0 LATX_KZT=0 LATX_VPAES=0 \ + "$@" "$emulator" "$guest" +} + +for guest in "$workdir/rip-static" "$workdir/rip-pie" \ + "$workdir/rip-shared"; do + "$guest" + run_guest "$guest" LATX_IMM_RIP=0 + run_guest "$guest" LATX_IMM_RIP=1 LATX_IMM_RIP_STATS=1 \ + 2>>"$workdir/enabled.err" +done + +if run_guest "$workdir/rip-static" LATX_IMM_RIP=invalid >/dev/null 2>&1; then + echo "FAIL: invalid LATX_IMM_RIP was accepted" >&2 + exit 1 +fi +if run_guest "$workdir/rip-static" LATX_IMM_RIP_STATS=2 \ + >/dev/null 2>&1; then + echo "FAIL: invalid LATX_IMM_RIP_STATS was accepted" >&2 + exit 1 +fi + +run_guest "$workdir/rip-static" LATX_IMM_RIP=0 LATX_IMM_RIP_STATS=1 \ + 2>"$workdir/disabled.err" +if grep -q '^\[LATX\]\[imm-rip\]' "$workdir/disabled.err"; then + echo "FAIL: disabled RIP cache produced statistics" >&2 + exit 1 +fi + +grep -Eq '^\[LATX\]\[imm-rip\].*calls=[1-9][0-9]*.*hits=[1-9][0-9]*' \ + "$workdir/enabled.err" +grep -Eq '^\[LATX\]\[imm-rip\].*misses=[1-9][0-9]*' \ + "$workdir/enabled.err" + +LATX_IMM_RIP=1 LATX_IMM_RIP_STATS=1 \ + timeout 10s "$emulator" -latx-imm-rip 0 \ + "$workdir/rip-static" 2>"$workdir/override.err" +if grep -q '^\[LATX\]\[imm-rip\]' "$workdir/override.err"; then + echo "FAIL: command-line disable did not override the environment" >&2 + exit 1 +fi + +echo "PASS: RIP cache ELF, GOT, page, helper, signal, fork/exec, and SMC matrix" diff --git a/tests/latx/latx-config-regression.c b/tests/latx/latx-config-regression.c index 2bbc14b2a23..9a0f48d4b18 100644 --- a/tests/latx/latx-config-regression.c +++ b/tests/latx/latx-config-regression.c @@ -39,6 +39,18 @@ static void test_release_loader_prefix_config(void) g_assert_true(config_option_registered("LAT_LD_PREFIX")); } +static void test_rip_imm_cache_options_registered(void) +{ + g_assert_true(config_option_registered("LATX_IMM_RIP")); + g_assert_true(config_option_registered("LATX_IMM_RIP_STATS")); +} + +static void test_complex_imm_cache_options_registered(void) +{ + g_assert_true(config_option_registered("LATX_IMM_COMPLEX")); + g_assert_true(config_option_registered("LATX_IMM_COMPLEX_STATS")); +} + static void test_runtime_prefix_source(void) { latx_runtime_reset(); @@ -187,6 +199,10 @@ int main(int argc, char **argv) test_target_config_files); g_test_add_func("/latx/config/release-loader-prefix", test_release_loader_prefix_config); + g_test_add_func("/latx/config/rip-imm-cache-options", + test_rip_imm_cache_options_registered); + g_test_add_func("/latx/config/complex-imm-cache-options", + test_complex_imm_cache_options_registered); g_test_add_func("/latx/config/runtime-prefix-source", test_runtime_prefix_source); g_test_add_func("/latx/config/user-config-path",