LATX: optimize stack updates and share helper stubs

accel/tcg/translate-all.c

@@ -474,6 +474,18 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
         cpu_neg(cpu)->icount_decr.u16.low += num_insns - i;
     }
     restore_state_to_opc(env, tb, data);
+#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS
+    target_long sp_delta = (target_long)data[1];
+#ifdef TARGET_X86_64
+    if (tb->bool_flags & IS_CODE64) {
+        env->regs[R_ESP] += sp_delta;
+    } else {
+        env->regs[R_ESP] = (uint32_t)(env->regs[R_ESP] + sp_delta);
+    }
+#else
+    env->regs[R_ESP] += sp_delta;
+#endif
+#endif
 #ifdef CONFIG_LATX_OPT_PUSH_POP
     uint32_t parallel = cpu->tcg_cflags & CF_PARALLEL;
     if (!parallel) {

linux-user/main.c

@@ -635,6 +635,31 @@ static void handle_arg_latx_vpaes(const char *arg)
     option_vpaes = strtol(arg, NULL, 0);
 }
 
+static void handle_arg_latx_static_helper(const char *arg)
+{
+    int value;
+
+    if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) {
+        error_report("LATX_STATIC_HELPER must be exactly 0 or 1 (got '%s')",
+                     arg);
+        exit(EXIT_FAILURE);
+    }
+    option_static_helper = value;
+}
+
+static void handle_arg_latx_static_helper_stats(const char *arg)
+{
+    int value;
+
+    if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) {
+        error_report(
+            "LATX_STATIC_HELPER_STATS must be exactly 0 or 1 (got '%s')",
+            arg);
+        exit(EXIT_FAILURE);
+    }
+    option_static_helper_stats = value;
+}
+
 static void handle_arg_latx_parallel(const char *arg)
 {
     close_latx_parallel = strtol(arg, NULL, 0);
@@ -906,6 +931,12 @@ static const struct qemu_argument arg_table[] = {
     "",           "specify enabled optimize type"},
     {"latx-vpaes",      "LATX_VPAES",         true,  handle_arg_latx_vpaes,
     "",           "enable vpaes AES translation"},
+    {"latx-static-helper", "LATX_STATIC_HELPER", true,
+    handle_arg_latx_static_helper, "0|1",
+    "use a shared save/call/restore stub for selected helpers"},
+    {"latx-static-helper-stats", "LATX_STATIC_HELPER_STATS", true,
+    handle_arg_latx_static_helper_stats, "0|1",
+    "report translated callsites for static helper stubs"},
     {"latx-smc",        "LATX_SMC",         true,   handle_arg_latx_smc,
     "",           "smc strategy: 0 (page) 1 (tb) 2(+shmm) 6(+helper,default)"},
     {"latx-close-parallel",    "LATX_CLOSE_PARALLEL",     true,  handle_arg_latx_parallel,

target/i386/latx/include/aot.h

@@ -39,9 +39,9 @@ extern  const char *aot_left_file_minsize_optarg;
  * +--------------+
  */
 #ifdef CONFIG_LATX_DEBUG
-#define AOT_VERSION "Version: "LATX_VERSION"-debug"
+#define AOT_VERSION "Version: "LATX_VERSION"-debug-shcfg1"
 #else
-#define AOT_VERSION "Version: "LATX_VERSION"-release"
+#define AOT_VERSION "Version: "LATX_VERSION"-release-shcfg1"
 #endif
 typedef struct aot_header {
     uint32_t lib_size;
@@ -57,6 +57,7 @@ typedef struct aot_header {
 #define CACHE_AOT_FILE      4
 #define HASH_AOT_FILE       8
     uint8_t aot_file_type;
+    uint8_t static_helper;
 } aot_header;
 
 typedef struct aot_file_info {
@@ -260,6 +261,10 @@ typedef enum aot_rel_kind {
     LOAD_HELPER_CVTPH2PS_XMM,
     LOAD_HELPER_CVTPS2PH_YMM,
     LOAD_HELPER_CVTPS2PH_XMM,
+    LOAD_STATIC_HELPER_PROLOGUE,
+    LOAD_STATIC_HELPER_EPILOGUE,
+    LOAD_STATIC_HELPER_NOFP_PROLOGUE,
+    LOAD_STATIC_HELPER_NOFP_EPILOGUE,
 
     LOAD_HELPER_END,
 

target/i386/latx/include/env.h

@@ -26,6 +26,9 @@ typedef struct TRANSLATION_DATA {
     /* ir1 */
     IR1_INST *curr_ir1_inst;
     int curr_ir1_count;
+#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS
+    int sp_delta;
+#endif
     /* uint8       ir1_dump_threshold[MAX_IR1_NUM_PER_TB]; */
 
     /* ir2 */

target/i386/latx/include/latx-options.h

@@ -89,6 +89,8 @@ extern int option_fputag;
 extern int option_save_xmm;
 extern int option_enable_lasx;
 extern int option_vpaes;
+extern int option_static_helper;
+extern int option_static_helper_stats;
 extern int option_split_tb;
 extern int option_anonym;
 extern int option_imm_reg;
@@ -125,6 +127,8 @@ extern unsigned long long counter_mips_tr;
 #define ENVSUP_LATX \
     ENVFUN(LATX_OPTIMIZE, handle_arg_optimize) \
     ENVFUN(LATX_VPAES, handle_arg_latx_vpaes) \
+    ENVFUN(LATX_STATIC_HELPER, handle_arg_latx_static_helper) \
+    ENVFUN(LATX_STATIC_HELPER_STATS, handle_arg_latx_static_helper_stats) \
     ENVFUN(LATX_SMC, handle_arg_latx_smc) \
     ENVFUN(LATX_CLOSE_PARALLEL, handle_arg_latx_parallel) \
     ENVFUN(LATX_SOFTFPU, handle_arg_latx_softfpu) \

target/i386/latx/include/optimize-config.h

@@ -78,7 +78,12 @@
 #undef CONFIG_LATX_JRRA
 #define CONFIG_LATX_JRRA            /* jr-ra, */
 #undef CONFIG_LATX_OPT_PUSH_POP
-#define CONFIG_LATX_OPT_PUSH_POP
+#undef CONFIG_LATX_OPT_PUSH_POP_TRANS
+#define CONFIG_LATX_OPT_PUSH_POP_TRANS
+#if defined(CONFIG_LATX_OPT_PUSH_POP) && \
+    defined(CONFIG_LATX_OPT_PUSH_POP_TRANS)
+#error "PUSH/POP optimizations must be mutually exclusive"
+#endif
 #undef CONFIG_LATX_SSSE3_SSE4
 #define CONFIG_LATX_SSSE3_SSE4
 #undef CONFIG_LATX_SPLIT_TB

target/i386/latx/include/translate.h

@@ -1502,6 +1502,10 @@ void tr_disasm(struct TranslationBlock *tb, int max_insns);
 void etb_add_succ(void* etb,int depth);
 int tr_translate_tb(struct TranslationBlock *tb);
 int tr_ir2_generate(struct TranslationBlock *tb);
+#ifdef CONFIG_LATX_OPT_PUSH_POP_TRANS
+bool latx_sp_merge_can_delay(IR1_INST *pir1);
+void latx_sp_merge_flush(void);
+#endif
 int label_dispose(TranslationBlock *tb, TRANSLATION_DATA *lat_ctx);
 int tr_ir2_assemble(const void *code_start_addr, const IR2_INST *pir2);
 #if defined(CONFIG_LATX_FLAG_REDUCTION) && \
@@ -1706,6 +1710,10 @@ void tr_generate_goto_tb(void);                          /* TODO */
 extern ADDR native_rotate_fpu_by;
 extern ADDR indirect_jmp_glue;
 extern ADDR parallel_indirect_jmp_glue;
+extern ADDR static_helper_prologue;
+extern ADDR static_helper_epilogue;
+extern ADDR static_helper_nofp_prologue;
+extern ADDR static_helper_nofp_epilogue;
 void rotate_fpu_to_top(int top);
 void rotate_fpu_by(int step);
 void rotate_fpu_to_bias(int bias);
@@ -1720,6 +1728,8 @@ void tr_gen_call_to_helper_cvttpd2pi(ADDR, int, int, enum aot_rel_kind);
 void tr_gen_call_to_helper_pclmulqdq(ADDR, int, int, int, int ,int,
         enum aot_rel_kind );
 void tr_gen_call_to_helper_aes(ADDR, int, int, int, enum aot_rel_kind);
+bool tr_gen_call_to_static_helper_nofp_prologue(IR2_OPND live_opnd);
+void tr_gen_call_to_static_helper_nofp_epilogue(void);
 void tr_load_top_from_env(void);
 void tr_gen_top_mode_init(void);
 

target/i386/latx/latx-options.c

@@ -97,6 +97,8 @@ int option_fputag;
 int option_save_xmm;
 int option_enable_lasx;
 int option_vpaes;
+int option_static_helper;
+int option_static_helper_stats;
 int option_split_tb;
 int option_anonym;
 int option_mem_test;
@@ -236,6 +238,8 @@ void options_init(void)
     option_latx_disassemble_trace_cmp = 0;
     option_enable_lasx = 1;
     option_vpaes = 0;
+    option_static_helper = 1;
+    option_static_helper_stats = 0;
 
     counter_tb_exec = 0;
     counter_tb_tr = 0;

target/i386/latx/optimization/ir2-optimization.c

@@ -215,6 +215,7 @@ void tri_avoid_leading_label(void)
 #endif
 }
 
+#ifdef CONFIG_LATX_OPT_PUSH_POP
 static int ir2_get_addi_rsp_offs(IR2_INST *ir2)
 {
     int rsp = reg_gpr_map[esp_index];
@@ -435,7 +436,6 @@ static void ir2_opt_push_pop(TranslationBlock *tb)
     }
     IR2_INST *curr;
     int curr_id, end_id, off, patch_off;
-    //int branch_id = 0;
     bool insert;
     IR2_OPCODE op;
 
@@ -525,6 +525,7 @@ static void ir2_opt_push_pop(TranslationBlock *tb)
                     "patch_off = %-5d\n", curr_id, 0);
     }
 }
+#endif
 
 void tr_ir2_optimize(TranslationBlock *tb)
 {
@@ -545,6 +546,7 @@ void tr_ir2_optimize(TranslationBlock *tb)
     } */
 }
 
+#ifdef CONFIG_LATX_OPT_PUSH_POP
 extern void ir1_optimization(TranslationBlock *tb);
 
 void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i)
@@ -704,3 +706,4 @@ void ir2_opt_push_pop_fix(TranslationBlock *tb, CPUState *cpu, int i)
     env->regs[R_ESP] += patch_off;
     tr_fini(false);
 }
+#endif

target/i386/latx/sbt/aot.c

@@ -766,6 +766,7 @@ int do_generate_aot(int first_seg_in_lib, int end_seg_in_lib)
 
     p_header->aot_file_type =
         seg_info_vector[first_seg_in_lib]->aot_file_type;
+    p_header->static_helper = !!option_static_helper;
 
     if (p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE)) {
         struct stat statbuf;
@@ -1298,8 +1299,6 @@ static void* relkind_to_fixup_addr[] = {
     [LOAD_HELPER_CVTPS2PH_YMM] = helper_cvtps2ph_ymm,
     [LOAD_HELPER_CVTPS2PH_XMM] = helper_cvtps2ph_xmm,
 #endif
-
-
 };
 
 void aot_do_tb_reloc(TranslationBlock *tb, struct aot_tb *stb,
@@ -1418,7 +1417,21 @@ void aot_do_tb_reloc(TranslationBlock *tb, struct aot_tb *stb,
             }
             break;
         case LOAD_HELPER_BEGIN ... LOAD_HELPER_END:
-            helper_address = (uintptr_t)relkind_to_fixup_addr[aot_rel_table[i].kind];
+            if (aot_rel_table[i].kind == LOAD_STATIC_HELPER_PROLOGUE) {
+                helper_address = static_helper_prologue;
+            } else if (aot_rel_table[i].kind ==
+                       LOAD_STATIC_HELPER_EPILOGUE) {
+                helper_address = static_helper_epilogue;
+            } else if (aot_rel_table[i].kind ==
+                       LOAD_STATIC_HELPER_NOFP_PROLOGUE) {
+                helper_address = static_helper_nofp_prologue;
+            } else if (aot_rel_table[i].kind ==
+                       LOAD_STATIC_HELPER_NOFP_EPILOGUE) {
+                helper_address = static_helper_nofp_epilogue;
+            } else {
+                helper_address = (uintptr_t)
+                    relkind_to_fixup_addr[aot_rel_table[i].kind];
+            }
             lsassert(helper_address);
             lsassert((*pinsn & 0xfe000000) == 0x14000000); /* lu12i.w */
             *pinsn &= 0xfe00001f;

target/i386/latx/sbt/aot_merge.c

@@ -464,6 +464,7 @@ static bool merge_aot_generate(void)
     char *curr_name = aot_x86_lib_names;
     uint8_t aot_file_type = get_file_type(merge_seg_info_vector[0]->file_name);
     p_header->aot_file_type = aot_file_type;
+    p_header->static_helper = !!option_static_helper;
     int page_index = 0;
     for (int i = 0; i < seg_info_num; i++) {
         seg_info *curr_seg_info = merge_seg_info_vector[i]->s_info;
@@ -744,7 +745,8 @@ static AOTLoadResult aot_load_no_lock(char *lib_name)
             fclose(pf);
             goto load_error;
         }
-        if ((size_t)file_end < strlen(AOT_VERSION)) {
+        if ((size_t)file_end < sizeof(aot_header) ||
+            (size_t)file_end < strlen(AOT_VERSION)) {
             fclose(pf);
             if (i == 0) {
                 goto invalid_base;
@@ -788,6 +790,18 @@ static AOTLoadResult aot_load_no_lock(char *lib_name)
             fclose(pf);
             goto load_error;
         }
+        if (((aot_header *)buffer)->static_helper !=
+            !!option_static_helper) {
+            qemu_log_mask(LAT_LOG_AOT,
+                          "static helper mode changed, reject aot %s\n",
+                          path);
+            free(buffer);
+            fclose(pf);
+            if (i == 0) {
+                goto invalid_base;
+            }
+            goto load_error;
+        }
         fclose(pf);
         aot_buffer_all[j].p = buffer;
         aot_buffer_all[j].maplen = file_sz;

target/i386/latx/sbt/aot_reader.c

@@ -8,6 +8,7 @@
 #include "aot.h"
 #include "aot_reader.h"
 #include "file_ctx.h"
+#include "latx-options.h"
 #include "qemu.h"
 
 #ifdef CONFIG_LATX_AOT
@@ -65,6 +66,14 @@ int aot_get_tb_num(char *lib_name, char *aot_file_name, CPUState *cpu)
     aot_header *p_header = (aot_header *)buffer;
     struct stat statbuf;
 
+    if (p_header->static_helper != !!option_static_helper) {
+        qemu_log_mask(LAT_LOG_AOT,
+                      "static helper mode changed, remove aot %s\n",
+                      aot_file_path);
+        remove(aot_file_path);
+        goto out;
+    }
+
     if ((p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE))
             && (stat(lib_name, &statbuf)
             || p_header->lib_size != statbuf.st_size
@@ -157,6 +166,14 @@ lib_info *aot_load(char *lib_name, char *aot_file_name,
     assert(buffer);
     aot_header *p_header = (aot_header *)buffer;
 
+    if (p_header->static_helper != !!option_static_helper) {
+        qemu_log_mask(LAT_LOG_AOT,
+                      "static helper mode changed, remove aot %s\n",
+                      aot_file_path);
+        remove_curr_aot_file(fd);
+        goto out;
+    }
+
     if (p_header->aot_file_type & (ELF_AOT_FILE | PE_AOT_FILE)) {
         if (stat(lib_name, &statbuf)
                 || p_header->lib_size != statbuf.st_size

target/i386/latx/sbt/tests/aot-cache-reader-test.c

@@ -12,6 +12,7 @@ char aot_file_lock_buffer[PATH_MAX];
 char *aot_file_path = aot_file_path_buffer;
 char *aot_file_lock = aot_file_lock_buffer;
 int qemu_loglevel;
+int option_static_helper = 1;
 static bool fail_fdopen;
 static int tracked_fd;
 static int sentinel_fd;
@@ -102,6 +103,7 @@ static void write_cache(const char *name, bool has_header, bool has_footer,
     contents = g_malloc0(size);
     if (has_header) {
         ((aot_header *)contents)->aot_file_type = CACHE_AOT_FILE;
+        ((aot_header *)contents)->static_helper = !!option_static_helper;
     }
     if (has_footer) {
         memcpy(contents + size - footer_size, AOT_VERSION, footer_size);
@@ -134,6 +136,7 @@ int main(void)
     char bad_footer_name[] = "bad-footer";
     char truncated_name[] = "truncated";
     char complete_name[] = "complete";
+    char mode_mismatch_name[] = "mode-mismatch";
     char fdopen_failure_name[] = "fdopen-failure";
     char cache_path[PATH_MAX];
     void *buffer;
@@ -171,6 +174,20 @@ int main(void)
     assert_stream_closed();
     g_assert(lib_tree_remove(complete_name));
 
+    option_static_helper = 0;
+    write_cache(mode_mismatch_name, true, true, cache_path);
+    option_static_helper = 1;
+    g_assert(aot_get_tb_num(lib_name, mode_mismatch_name, NULL) == 0);
+    g_assert(!g_file_test(cache_path, G_FILE_TEST_EXISTS));
+
+    buffer = NULL;
+    write_cache(mode_mismatch_name, true, true, cache_path);
+    option_static_helper = 0;
+    g_assert(aot_load(lib_name, mode_mismatch_name, &buffer) == NULL);
+    g_assert(buffer == NULL);
+    g_assert(!g_file_test(cache_path, G_FILE_TEST_EXISTS));
+    option_static_helper = 1;
+
     reset_stream_counts();
     fail_fdopen = true;
     buffer = NULL;
@@ -184,6 +201,7 @@ int main(void)
     remove_cache(bad_footer_name);
     remove_cache(truncated_name);
     remove_cache(complete_name);
+    remove_cache(mode_mismatch_name);
     remove_cache(fdopen_failure_name);
     cache_dir = g_build_filename(test_dir, ".cache", "latx", NULL);
     g_assert(g_rmdir(cache_dir) == 0);