LATX: translate AES and VAES encryption rounds directly

configure

@@ -5612,6 +5612,9 @@ if !(GIT="$git" "$source_path/scripts/git-submodule.sh" "$git_submodules_action"
 fi
 
 config_host_mak="config-host.mak"
+# Keep one build timestamp for all translation units so AOT versions match.
+latx_build_timestamp=$(date -u +%Y%m%d%H%M%S)
+QEMU_CFLAGS="$QEMU_CFLAGS -DLATX_BUILD_TIMESTAMP=$latx_build_timestamp"
 
 echo "# Automatically generated by configure - do not modify" > $config_host_mak
 echo >> $config_host_mak

crypto/aes.c

@@ -28,6 +28,7 @@
  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "qemu/osdep.h"
+#include "qemu/bswap.h"
 #include "crypto/aes.h"
 
 typedef uint32_t u32;
@@ -660,6 +661,43 @@ const uint32_t AES_Te3[256] = {
     0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
     0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
 };
+
+uint32_t AES_Te_latx[256][4];
+uint32_t AES_sbox_latx[256][4];
+
+#define AES_VPAES_DUP16(...) { __VA_ARGS__, __VA_ARGS__ }
+const uint8_t AES_vpaes_lasx[7][32] __attribute__((aligned(32))) = {
+    AES_VPAES_DUP16(0x10,0x01,0x08,0x0d,0x0f,0x06,0x05,0x0e,
+                    0x02,0x0c,0x0b,0x0a,0x09,0x03,0x07,0x04),
+    AES_VPAES_DUP16(0x10,0x07,0x0b,0x0f,0x06,0x0a,0x04,0x01,
+                    0x09,0x08,0x05,0x02,0x0c,0x0e,0x0d,0x03),
+    AES_VPAES_DUP16(0x00,0x70,0x2a,0x5a,0x98,0xe8,0xb2,0xc2,
+                    0x08,0x78,0x22,0x52,0x90,0xe0,0xba,0xca),
+    AES_VPAES_DUP16(0x00,0x4d,0x7c,0x31,0x7d,0x30,0x01,0x4c,
+                    0x81,0xcc,0xfd,0xb0,0xfc,0xb1,0x80,0xcd),
+    AES_VPAES_DUP16(0x00,0xc7,0xbd,0x6f,0x17,0x6d,0xd2,0xd0,
+                    0x78,0xa8,0x02,0xc5,0x7a,0xbf,0xaa,0x15),
+    AES_VPAES_DUP16(0x00,0x6a,0xbb,0x5f,0xa5,0x74,0xe4,0xcf,
+                    0xfa,0x35,0x2b,0x41,0xd1,0x90,0x1e,0x8e),
+    AES_VPAES_DUP16(0x00,0x05,0x0a,0x0f,0x04,0x09,0x0e,0x03,
+                    0x08,0x0d,0x02,0x07,0x0c,0x01,0x06,0x0b),
+};
+#undef AES_VPAES_DUP16
+
+static void __attribute__((constructor)) aes_init_latx_te(void)
+{
+    for (int i = 0; i < 256; i++) {
+        AES_Te_latx[i][0] = bswap32(AES_Te0[i]);
+        AES_Te_latx[i][1] = bswap32(AES_Te1[i]);
+        AES_Te_latx[i][2] = bswap32(AES_Te2[i]);
+        AES_Te_latx[i][3] = bswap32(AES_Te3[i]);
+        AES_sbox_latx[i][0] = (uint32_t)AES_sbox[i];
+        AES_sbox_latx[i][1] = (uint32_t)AES_sbox[i] << 8;
+        AES_sbox_latx[i][2] = (uint32_t)AES_sbox[i] << 16;
+        AES_sbox_latx[i][3] = (uint32_t)AES_sbox[i] << 24;
+    }
+}
+
 const uint32_t AES_Te4[256] = {
     0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
     0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,

include/crypto/aes.h

@@ -60,4 +60,9 @@ extern const uint32_t AES_Te0[256], AES_Te1[256], AES_Te2[256],
 extern const uint32_t AES_Td0[256], AES_Td1[256], AES_Td2[256],
                       AES_Td3[256], AES_Td4[256];
 
+/* Pre-byte-swapped, interleaved encryption tables for LATX AES IR2. */
+extern uint32_t AES_Te_latx[256][4];
+extern uint32_t AES_sbox_latx[256][4];
+extern const uint8_t AES_vpaes_lasx[7][32];
+
 #endif

linux-user/syscall.c

@@ -12770,6 +12770,7 @@ CPUID_GROUP_EXT,
 CPUID_GROUP_EXT2,
 CPUID_GROUP_EXT3,
 CPUID_GROUP_EBX,
+CPUID_GROUP_ECX,
 CPUID_GROUP_XSAVE,
 };
 const char *__x86_cpuid_flags_table[] = {
@@ -12800,6 +12801,11 @@ const char *__x86_cpuid_flags_table[] = {
 "avx512f", "avx512dq", "rdseed","adx", "smap", "avx512ifma", "pcommit", "clflushopt",
 "clwb", "intel_pt", "avx512pf", "avx512er", "avx512cd", "sha_ni", "avx512bw", "avx512vl",
 /*xsave*/
+"", "avx512vbmi", "umip", "pku", "ospke", "waitpkg", "avx512vbmi2", "",
+"gfni", "vaes", "vpclmulqdq", "avx512vnni", "avx512bitalg", "", "avx512-vpopcntdq", "",
+"la57", "", "", "", "", "", "rdpid", "",
+"bus-lock-detect", "cldemote", "", "movdiri", "movdir64b", "", "", "pks",
+/*xsave*/
 "xsaveopt", "xsavec", "xgetbv1", "xsaves", "", "", "", "",
 "", "", "", "", "" "", "", "",
 "", "", "", "", "", "", "", "",
@@ -12825,6 +12831,8 @@ static char *cpuinfo_flags(CPUArchState *env)
                 group = CPUID_GROUP_EXT3;
             } else if (i == FEAT_7_0_EBX) {
                 group = CPUID_GROUP_EBX;
+            } else if (i == FEAT_7_0_ECX) {
+                group = CPUID_GROUP_ECX;
             } else if (i == FEAT_XSAVE) {
                 group = CPUID_GROUP_XSAVE;
             } else if (i == FEAT_XSAVE_COMP_LO || i ==FEAT_XSAVE_COMP_HI) {

target/i386/cpu.c

@@ -687,7 +687,7 @@ static void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1,
 #ifdef CONFIG_LATX_AVX_OPT
 #define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_PKU | \
           /* CPUID_7_0_ECX_OSPKE is dynamic */ \
-          CPUID_7_0_ECX_LA57)
+          CPUID_7_0_ECX_LA57 | CPUID_7_0_ECX_VAES)
 #else/*CONFIG_LATX_AVX_OPT*/
 #define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_PKU | \
           /* CPUID_7_0_ECX_OSPKE is dynamic */ \
@@ -1933,6 +1933,8 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_AVX2 |
             CPUID_7_0_EBX_BMI2,
+        .features[FEAT_7_0_ECX] =
+            CPUID_7_0_ECX_VAES,
         .features[FEAT_XSAVE] =
             CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XGETBV1,
 #endif/*CONFIG_LATX_AVX_OPT*/
@@ -2081,6 +2083,8 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_AVX2 |
             CPUID_7_0_EBX_BMI2,
+        .features[FEAT_7_0_ECX] =
+            CPUID_7_0_ECX_VAES,
         .features[FEAT_XSAVE] =
             CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XGETBV1,
 #endif

target/i386/latx/include/aot.h

@@ -39,10 +39,13 @@ extern  const char *aot_left_file_minsize_optarg;
  * +--------------+
  */
 #ifdef CONFIG_LATX_DEBUG
-#define AOT_VERSION "Version: "LATX_VERSION"-debug"
+#define AOT_BUILD_FLAVOR "debug"
 #else
-#define AOT_VERSION "Version: "LATX_VERSION"-release"
+#define AOT_BUILD_FLAVOR "release"
 #endif
+#define LATX_STRINGIFY_INNER(value) #value
+#define LATX_STRINGIFY(value) LATX_STRINGIFY_INNER(value)
+#define AOT_VERSION "Version: " LATX_VERSION "-" AOT_BUILD_FLAVOR "-" LATX_STRINGIFY(LATX_BUILD_TIMESTAMP)
 typedef struct aot_header {
     uint32_t lib_size;
     struct timespec last_modify_time;
@@ -230,8 +233,9 @@ typedef enum aot_rel_kind {
     LOAD_HELPER_AESKEYGENASSIST_XMM,
     LOAD_HELPER_AESDEC_XMM,
     LOAD_HELPER_AESDECLAST_XMM,
-    LOAD_HELPER_AESENC_XMM,
-    LOAD_HELPER_AESENCLAST_XMM,
+    LOAD_HOST_AES_TE_LATX,
+    LOAD_HOST_AES_SBOX_LATX,
+    LOAD_HOST_AES_VPAES_LASX,
     LOAD_HELPER_SHA1NEXTE,
     LOAD_HELPER_SHA1MSG1,
     LOAD_HELPER_SHA1MSG2,

target/i386/latx/include/env.h

@@ -64,6 +64,10 @@ typedef struct TRANSLATION_DATA {
 
     bool softfpu_region_active;
 
+    /* Read-only AES table address retained across consecutive AES rounds. */
+    int aes_table_reg;
+    uint8_t aes_table_kind;
+
     /* manage the immediate number cache */
     IMM_CACHE *imm_cache;
 } TRANSLATION_DATA;

target/i386/latx/include/translate.h

@@ -1447,6 +1447,13 @@ TRANS_FUNC_DEF(aesenclast);
 TRANS_FUNC_DEF(aesimc);
 TRANS_FUNC_DEF(aeskeygenassist);
 
+void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key);
+void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key);
+void gen_aesenc_ir2_no_key(IR2_OPND dest);
+void gen_aesenclast_ir2_no_key(IR2_OPND dest);
+void gen_vaesenc_lasx_ir2(IR2_OPND dest, IR2_OPND key, bool is_last);
+void load_aes_key_from_mem(IR2_OPND key, IR1_OPND *mem);
+
 #ifdef CONFIG_LATX_AVX_OPT
 TRANS_FUNC_DEF(vaesdec);
 TRANS_FUNC_DEF(vaesdeclast);

target/i386/latx/sbt/aot.c

@@ -28,6 +28,7 @@
 #include "aot_smc.h"
 #include "aot_page.h"
 #include "../translator/tr-vpaes.h"
+#include "crypto/aes.h"
 #include<sys/syscall.h>
 #include "exec/translate-all.h"
 #include "latx-smc.h"
@@ -1266,8 +1267,9 @@ static void* relkind_to_fixup_addr[] = {
     [LOAD_HELPER_AESKEYGENASSIST_XMM] = helper_aeskeygenassist_xmm,
     [LOAD_HELPER_AESDEC_XMM] = helper_aesdec_xmm,
     [LOAD_HELPER_AESDECLAST_XMM] = helper_aesdeclast_xmm,
-    [LOAD_HELPER_AESENC_XMM] = helper_aesenc_xmm,
-    [LOAD_HELPER_AESENCLAST_XMM] = helper_aesenclast_xmm,
+    [LOAD_HOST_AES_TE_LATX] = (void *)AES_Te_latx,
+    [LOAD_HOST_AES_SBOX_LATX] = (void *)AES_sbox_latx,
+    [LOAD_HOST_AES_VPAES_LASX] = (void *)AES_vpaes_lasx,
     [LOAD_HELPER_SHA1NEXTE] = helper_sha1nexte,
     [LOAD_HELPER_SHA1MSG1] = helper_sha1msg1,
     [LOAD_HELPER_SHA1MSG2] = helper_sha1msg2,

target/i386/latx/translator/tr-avx.c

@@ -5900,6 +5900,54 @@ bool translate_vaesdeclast(IR1_INST *pir1)
     return true;
 }
 
+static bool translate_vaes_xmm_ir2(IR1_INST *pir1, bool is_last)
+{
+    IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0);
+    IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1);
+    IR1_OPND *opnd2 = ir1_get_opnd(pir1, 2);
+    int d = ir1_opnd_base_reg_num(opnd0);
+    int s1 = ir1_opnd_base_reg_num(opnd1);
+    IR2_OPND dest = ra_alloc_xmm(d);
+    IR2_OPND src1 = ra_alloc_xmm(s1);
+    IR2_OPND key;
+    bool key_is_temp = false;
+
+    lsassert(ir1_opnd_is_xmm(opnd0));
+    lsassert(ir1_opnd_is_xmm(opnd1));
+
+    if (ir1_opnd_is_mem(opnd2)) {
+        key = ra_alloc_ftemp();
+        key_is_temp = true;
+        load_aes_key_from_mem(key, opnd2);
+    } else {
+        int s2 = ir1_opnd_base_reg_num(opnd2);
+        IR2_OPND src2 = ra_alloc_xmm(s2);
+
+        if (s2 == d) {
+            key = ra_alloc_ftemp();
+            key_is_temp = true;
+            la_vor_v(key, src2, src2);
+        } else {
+            key = src2;
+        }
+    }
+
+    if (d != s1) {
+        la_vor_v(dest, src1, src1);
+    }
+    if (is_last) {
+        gen_aesenclast_ir2(dest, key);
+    } else {
+        gen_aesenc_ir2(dest, key);
+    }
+    set_high128_xreg_to_zero(dest);
+
+    if (key_is_temp) {
+        ra_free_temp(key);
+    }
+    return true;
+}
+
 bool translate_vaesenc(IR1_INST *pir1)
 {
     if (!option_enable_lasx) {
@@ -5916,46 +5964,35 @@ bool translate_vaesenc(IR1_INST *pir1)
     int d = ir1_opnd_base_reg_num(opnd0);
     int s1 = ir1_opnd_base_reg_num(opnd1);
 
-    ADDR helper_func;
-    int helper_kind;
-
     if (ir1_opnd_is_ymm(opnd0)) {
-        helper_func = (ADDR)helper_vaesenc_ymm;
-        helper_kind = LOAD_HELPER_VAESENC_YMM;
-    } else {
-        helper_func = (ADDR)helper_vaesenc_xmm;
-        helper_kind = LOAD_HELPER_VAESENC_XMM;
-    }
-
-    if (!ir1_opnd_is_mem(opnd2)) {
-        int s2 = ir1_opnd_base_reg_num(opnd2);
-        tr_gen_call_to_helper_aes((ADDR)helper_func, d, s1, s2,
-                helper_kind);
-    } else {
-        int s2 = 0;
-        while (s2 < 8) {
-            if (s2 != d && s2 != s1) {
-                break;
+        IR2_OPND dest = ra_alloc_xmm(d);
+        IR2_OPND src1 = load_freg256_from_ir1(opnd1);
+        IR2_OPND src2;
+        bool src2_is_temp = false;
+        if (ir1_opnd_is_mem(opnd2)) {
+            src2 = ra_alloc_ftemp();
+            load_freg256_from_ir1_mem(src2, opnd2);
+            src2_is_temp = true;
+        } else {
+            src2 = load_freg256_from_ir1(opnd2);
+            if (ir1_opnd_base_reg_num(opnd2) == d) {
+                IR2_OPND key = ra_alloc_ftemp();
+                la_xvor_v(key, src2, src2);
+                src2 = key;
+                src2_is_temp = true;
             }
-            s2++;
         }
-        IR2_OPND temp = ra_alloc_ftemp();
-        IR2_OPND src = ra_alloc_xmm(s2);
-        la_xvor_v(temp, src, src);
-        if (ir1_opnd_size(opnd2) == 128) {
-            load_freg128_from_ir1_mem(src, opnd2);
-        } else {
-            load_freg256_from_ir1_mem(src, opnd2);
+        if (d != s1) {
+            la_xvor_v(dest, src1, src1);
         }
-        tr_gen_call_to_helper_aes((ADDR)helper_func, d, s1, s2,
-                helper_kind);
-        la_xvor_v(src, temp, temp);
-    }
-    if (!ir1_opnd_is_ymm(opnd0)) {
-        set_high128_xreg_to_zero(ra_alloc_xmm(d));
+        gen_vaesenc_lasx_ir2(dest, src2, false);
+        if (src2_is_temp) {
+            ra_free_temp(src2);
+        }
+        return true;
     }
-    /* TODO: need to check */
-    return true;
+
+    return translate_vaes_xmm_ir2(pir1, false);
 }
 
 bool translate_vaesenclast(IR1_INST *pir1)
@@ -5974,46 +6011,35 @@ bool translate_vaesenclast(IR1_INST *pir1)
     int d = ir1_opnd_base_reg_num(opnd0);
     int s1 = ir1_opnd_base_reg_num(opnd1);
 
-    ADDR helper_func;
-    int helper_kind;
-
     if (ir1_opnd_is_ymm(opnd0)) {
-        helper_func = (ADDR)helper_vaesenclast_ymm;
-        helper_kind = LOAD_HELPER_VAESENCLAST_YMM;
-    } else {
-        helper_func = (ADDR)helper_vaesenclast_xmm;
-        helper_kind = LOAD_HELPER_VAESENCLAST_XMM;
-    }
-
-    if (!ir1_opnd_is_mem(opnd2)) {
-        int s2 = ir1_opnd_base_reg_num(opnd2);
-        tr_gen_call_to_helper_aes((ADDR)helper_func, d, s1, s2,
-                helper_kind);
-    } else {
-        int s2 = 0;
-        while (s2 < 8) {
-            if (s2 != d && s2 != s1) {
-                break;
+        IR2_OPND dest = ra_alloc_xmm(d);
+        IR2_OPND src1 = load_freg256_from_ir1(opnd1);
+        IR2_OPND src2;
+        bool src2_is_temp = false;
+        if (ir1_opnd_is_mem(opnd2)) {
+            src2 = ra_alloc_ftemp();
+            load_freg256_from_ir1_mem(src2, opnd2);
+            src2_is_temp = true;
+        } else {
+            src2 = load_freg256_from_ir1(opnd2);
+            if (ir1_opnd_base_reg_num(opnd2) == d) {
+                IR2_OPND key = ra_alloc_ftemp();
+                la_xvor_v(key, src2, src2);
+                src2 = key;
+                src2_is_temp = true;
             }
-            s2++;
         }
-        IR2_OPND temp = ra_alloc_ftemp();
-        IR2_OPND src = ra_alloc_xmm(s2);
-        la_xvor_v(temp, src, src);
-        if (ir1_opnd_size(opnd2) == 128) {
-            load_freg128_from_ir1_mem(src, opnd2);
-        } else {
-            load_freg256_from_ir1_mem(src, opnd2);
+        if (d != s1) {
+            la_xvor_v(dest, src1, src1);
         }
-        tr_gen_call_to_helper_aes((ADDR)helper_func, d, s1, s2,
-                helper_kind);
-        la_xvor_v(src, temp, temp);
-    }
-    if (!ir1_opnd_is_ymm(opnd0)) {
-        set_high128_xreg_to_zero(ra_alloc_xmm(d));
+        gen_vaesenc_lasx_ir2(dest, src2, true);
+        if (src2_is_temp) {
+            ra_free_temp(src2);
+        }
+        return true;
     }
-    /* TODO: need to check */
-    return true;
+
+    return translate_vaes_xmm_ir2(pir1, true);
 }
 
 bool translate_vaesimc(IR1_INST *pir1)