From 50e3dcf19f0563b8da516511d9d444605b8e7ffe Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 26 Aug 2026 10:51:28 +0800 Subject: [PATCH 1/6] LATX, aot: key caches by build identity Signed-off-by: Lu Zeng --- configure | 3 +++ target/i386/latx/include/aot.h | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/configure b/configure index e4c41093ac9..a396decd504 100755 --- a/configure +++ b/configure @@ -5612,6 +5612,9 @@ if !(GIT="$git" "$source_path/scripts/git-submodule.sh" "$git_submodules_action" fi config_host_mak="config-host.mak" +# Keep one build timestamp for all translation units so AOT versions match. +latx_build_timestamp=$(date -u +%Y%m%d%H%M%S) +QEMU_CFLAGS="$QEMU_CFLAGS -DLATX_BUILD_TIMESTAMP=$latx_build_timestamp" echo "# Automatically generated by configure - do not modify" > $config_host_mak echo >> $config_host_mak diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index f72613a814d..4456b6aa47f 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -39,10 +39,13 @@ extern const char *aot_left_file_minsize_optarg; * +--------------+ */ #ifdef CONFIG_LATX_DEBUG -#define AOT_VERSION "Version: "LATX_VERSION"-debug" +#define AOT_BUILD_FLAVOR "debug" #else -#define AOT_VERSION "Version: "LATX_VERSION"-release" +#define AOT_BUILD_FLAVOR "release" #endif +#define LATX_STRINGIFY_INNER(value) #value +#define LATX_STRINGIFY(value) LATX_STRINGIFY_INNER(value) +#define AOT_VERSION "Version: " LATX_VERSION "-" AOT_BUILD_FLAVOR "-" LATX_STRINGIFY(LATX_BUILD_TIMESTAMP) typedef struct aot_header { uint32_t lib_size; struct timespec last_modify_time; From d6669f599a0e5e1606d89a347b81d83ab6a088c0 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 26 Aug 2026 09:20:06 +0800 Subject: [PATCH 2/6] LATX, opt: translate AES encryption rounds directly Generate AESENC and AESENCLAST with scalar table-based IR2 instead of C helpers, and relocate the AES tables for AOT loading. Preserve memory and aliased round keys and release the consumed memory-address temporary before allocating the seven AES integer temporaries. This avoids corrupting the environment register on memory-source forms. Signed-off-by: Lu Zeng --- target/i386/latx/include/aot.h | 4 +- target/i386/latx/sbt/aot.c | 5 +- target/i386/latx/translator/tr-simd.c | 188 ++++++++++++++++++++------ 3 files changed, 153 insertions(+), 44 deletions(-) diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index 4456b6aa47f..a6e5d34ff1e 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -233,8 +233,8 @@ typedef enum aot_rel_kind { LOAD_HELPER_AESKEYGENASSIST_XMM, LOAD_HELPER_AESDEC_XMM, LOAD_HELPER_AESDECLAST_XMM, - LOAD_HELPER_AESENC_XMM, - LOAD_HELPER_AESENCLAST_XMM, + LOAD_HOST_AES_TE0, + LOAD_HOST_AES_SBOX, LOAD_HELPER_SHA1NEXTE, LOAD_HELPER_SHA1MSG1, LOAD_HELPER_SHA1MSG2, diff --git a/target/i386/latx/sbt/aot.c b/target/i386/latx/sbt/aot.c index 3aa6292cfac..7fec55fb7e5 100644 --- a/target/i386/latx/sbt/aot.c +++ b/target/i386/latx/sbt/aot.c @@ -28,6 +28,7 @@ #include "aot_smc.h" #include "aot_page.h" #include "../translator/tr-vpaes.h" +#include "crypto/aes.h" #include #include "exec/translate-all.h" #include "latx-smc.h" @@ -1266,8 +1267,8 @@ static void* relkind_to_fixup_addr[] = { [LOAD_HELPER_AESKEYGENASSIST_XMM] = helper_aeskeygenassist_xmm, [LOAD_HELPER_AESDEC_XMM] = helper_aesdec_xmm, [LOAD_HELPER_AESDECLAST_XMM] = helper_aesdeclast_xmm, - [LOAD_HELPER_AESENC_XMM] = helper_aesenc_xmm, - [LOAD_HELPER_AESENCLAST_XMM] = helper_aesenclast_xmm, + [LOAD_HOST_AES_TE0] = (void *)AES_Te0, + [LOAD_HOST_AES_SBOX] = (void *)AES_sbox, [LOAD_HELPER_SHA1NEXTE] = helper_sha1nexte, [LOAD_HELPER_SHA1MSG1] = helper_sha1msg1, [LOAD_HELPER_SHA1MSG2] = helper_sha1msg2, diff --git a/target/i386/latx/translator/tr-simd.c b/target/i386/latx/translator/tr-simd.c index 9e97cead690..8c39c75c5fa 100644 --- a/target/i386/latx/translator/tr-simd.c +++ b/target/i386/latx/translator/tr-simd.c @@ -11,6 +11,7 @@ #include "hbr.h" #include "tr-vpaes.h" #include "pclmul.h" +#include "crypto/aes.h" bool translate_por(IR1_INST *pir1) { @@ -3767,6 +3768,101 @@ bool translate_pclmulqdq(IR1_INST * pir1) { return true; } +static void aes_load_te0(IR2_OPND value, IR2_OPND index, + IR2_OPND table, IR2_OPND state, + int byte_shift, int rotate) +{ + la_bstrpick_d(index, state, byte_shift + 7, byte_shift); + /* IR2 ALSL encodes shift-1, so 1 means index * sizeof(uint32_t). */ + la_alsl_d(index, index, table, 1); + la_ld_wu(value, index, 0); + if (rotate) { + la_rotri_w(value, value, rotate); + } +} + +static void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) +{ + IR2_OPND state[4]; + IR2_OPND table = ra_alloc_itemp(); + IR2_OPND index = ra_alloc_itemp(); + IR2_OPND result = ra_alloc_itemp(); + + for (int i = 0; i < 4; i++) { + state[i] = ra_alloc_itemp(); + la_vpickve2gr_wu(state[i], dest, i); + } + aot_load_host_addr(table, (ADDR)AES_Te0, LOAD_HOST_AES_TE0, 0); + + for (int word = 0; word < 4; word++) { + aes_load_te0(result, index, table, state[word], 0, 0); + for (int column = 1; column < 4; column++) { + aes_load_te0(index, index, table, + state[(word + column) & 3], column * 8, + column * 8); + la_xor(result, result, index); + } + + /* bswap32 before XORing the round key, matching AESENC semantics. */ + la_revb_2h(result, result); + la_rotri_w(result, result, 16); + la_vpickve2gr_wu(index, key, word); + la_xor(result, result, index); + la_vinsgr2vr_w(dest, result, word); + } + + for (int i = 0; i < 4; i++) { + ra_free_temp(state[i]); + } + ra_free_temp(table); + ra_free_temp(index); + ra_free_temp(result); +} + +static void aes_load_sbox(IR2_OPND value, IR2_OPND index, + IR2_OPND table, IR2_OPND state, + int byte_shift) +{ + la_bstrpick_d(index, state, byte_shift + 7, byte_shift); + la_add_d(index, index, table); + la_ld_bu(value, index, 0); +} + +static void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key) +{ + IR2_OPND state[4]; + IR2_OPND table = ra_alloc_itemp(); + IR2_OPND index = ra_alloc_itemp(); + IR2_OPND result = ra_alloc_itemp(); + + for (int i = 0; i < 4; i++) { + state[i] = ra_alloc_itemp(); + la_vpickve2gr_wu(state[i], dest, i); + } + aot_load_host_addr(table, (ADDR)AES_sbox, LOAD_HOST_AES_SBOX, 0); + + for (int word = 0; word < 4; word++) { + aes_load_sbox(result, index, table, state[word], 0); + for (int byte = 1; byte < 4; byte++) { + aes_load_sbox(index, index, table, + state[(word + byte) & 3], byte * 8); + la_slli_w(index, index, byte * 8); + la_or(result, result, index); + } + + la_vpickve2gr_wu(index, key, word); + la_xor(result, result, index); + la_vinsgr2vr_w(dest, result, word); + } + + for (int i = 0; i < 4; i++) { + ra_free_temp(state[i]); + } + ra_free_temp(table); + ra_free_temp(index); + ra_free_temp(result); +} + bool translate_aesdec(IR1_INST *pir1) { if (option_vpaes) { @@ -3841,6 +3937,20 @@ bool translate_aesdeclast(IR1_INST *pir1) return true; } +static void load_aes_key_from_mem(IR2_OPND key, IR1_OPND *mem) +{ + int offset; + IR2_OPND address; + + assert(ir1_opnd_is_mem(mem)); + assert(ir1_opnd_size(mem) == 128); + address = convert_mem(mem, &offset); + gen_test_page_flag(address, offset, PAGE_READ); + la_vld(key, address, offset); + /* AES needs every x86-64 itemp; the emitted load no longer needs this. */ + ra_free_temp_auto(address); +} + bool translate_aesenc(IR1_INST *pir1) { if (option_vpaes) { @@ -3850,31 +3960,30 @@ bool translate_aesenc(IR1_INST *pir1) IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); int d = ir1_opnd_base_reg_num(opnd0); - if (!ir1_opnd_is_mem(opnd1)) { - int s1 = ir1_opnd_base_reg_num(opnd1); - tr_gen_call_to_helper_aes((ADDR)helper_aesenc_xmm, d, s1, 0, - LOAD_HELPER_AESENC_XMM); + IR2_OPND dest = ra_alloc_xmm(d); + IR2_OPND key; + bool key_is_temp = false; + + if (ir1_opnd_is_mem(opnd1)) { + key = ra_alloc_ftemp(); + key_is_temp = true; + load_aes_key_from_mem(key, opnd1); } else { - int s1 = (d + 1) & 7; - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND src = ra_alloc_xmm(s1); - if (option_enable_lasx) { - la_xvor_v(temp, src, src); + int s = ir1_opnd_base_reg_num(opnd1); + IR2_OPND src = ra_alloc_xmm(s); + if (s == d) { + key = ra_alloc_ftemp(); + key_is_temp = true; + la_vor_v(key, src, src); } else { - la_vor_v(temp, src, src); + key = src; } - assert(ir1_opnd_size(opnd1) == 128); - load_freg128_from_ir1_mem(src, opnd1); + } - tr_gen_call_to_helper_aes((ADDR)helper_aesenc_xmm, d, s1, 0, - LOAD_HELPER_AESENC_XMM); - if (option_enable_lasx) { - la_xvor_v(src, temp, temp); - } else { - la_vor_v(src, temp, temp); - } + gen_aesenc_ir2(dest, key); + if (key_is_temp) { + ra_free_temp(key); } - /* TODO: need to check */ return true; } @@ -3887,31 +3996,30 @@ bool translate_aesenclast(IR1_INST *pir1) IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); int d = ir1_opnd_base_reg_num(opnd0); - if (!ir1_opnd_is_mem(opnd1)) { - int s1 = ir1_opnd_base_reg_num(opnd1); - tr_gen_call_to_helper_aes((ADDR)helper_aesenclast_xmm, d, s1, 0, - LOAD_HELPER_AESENCLAST_XMM); + IR2_OPND dest = ra_alloc_xmm(d); + IR2_OPND key; + bool key_is_temp = false; + + if (ir1_opnd_is_mem(opnd1)) { + key = ra_alloc_ftemp(); + key_is_temp = true; + load_aes_key_from_mem(key, opnd1); } else { - int s1 = (d + 1) & 7; - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND src = ra_alloc_xmm(s1); - if (option_enable_lasx) { - la_xvor_v(temp, src, src); + int s = ir1_opnd_base_reg_num(opnd1); + IR2_OPND src = ra_alloc_xmm(s); + if (s == d) { + key = ra_alloc_ftemp(); + key_is_temp = true; + la_vor_v(key, src, src); } else { - la_vor_v(temp, src, src); + key = src; } - assert(ir1_opnd_size(opnd1) == 128); - load_freg128_from_ir1_mem(src, opnd1); + } - tr_gen_call_to_helper_aes((ADDR)helper_aesenclast_xmm, d, s1, 0, - LOAD_HELPER_AESENCLAST_XMM); - if (option_enable_lasx) { - la_xvor_v(src, temp, temp); - } else { - la_vor_v(src, temp, temp); - } + gen_aesenclast_ir2(dest, key); + if (key_is_temp) { + ra_free_temp(key); } - /* TODO: need to check */ return true; } From 7c38affbafaa04c9d25c2be3b4fea05ceb417dc2 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 26 Aug 2026 09:42:42 +0800 Subject: [PATCH 3/6] LATX, opt: optimize direct AES round generation Signed-off-by: Lu Zeng --- crypto/aes.c | 19 ++++++ include/crypto/aes.h | 4 ++ target/i386/latx/include/aot.h | 4 +- target/i386/latx/include/env.h | 4 ++ target/i386/latx/sbt/aot.c | 4 +- target/i386/latx/translator/tr-simd.c | 82 +++++++++++++++---------- target/i386/latx/translator/translate.c | 13 ++++ 7 files changed, 94 insertions(+), 36 deletions(-) diff --git a/crypto/aes.c b/crypto/aes.c index af72ff77799..78d837952f3 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -28,6 +28,7 @@ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "qemu/osdep.h" +#include "qemu/bswap.h" #include "crypto/aes.h" typedef uint32_t u32; @@ -660,6 +661,24 @@ const uint32_t AES_Te3[256] = { 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU, 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU, }; + +uint32_t AES_Te_latx[256][4]; +uint32_t AES_sbox_latx[256][4]; + +static void __attribute__((constructor)) aes_init_latx_te(void) +{ + for (int i = 0; i < 256; i++) { + AES_Te_latx[i][0] = bswap32(AES_Te0[i]); + AES_Te_latx[i][1] = bswap32(AES_Te1[i]); + AES_Te_latx[i][2] = bswap32(AES_Te2[i]); + AES_Te_latx[i][3] = bswap32(AES_Te3[i]); + AES_sbox_latx[i][0] = (uint32_t)AES_sbox[i]; + AES_sbox_latx[i][1] = (uint32_t)AES_sbox[i] << 8; + AES_sbox_latx[i][2] = (uint32_t)AES_sbox[i] << 16; + AES_sbox_latx[i][3] = (uint32_t)AES_sbox[i] << 24; + } +} + const uint32_t AES_Te4[256] = { 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, diff --git a/include/crypto/aes.h b/include/crypto/aes.h index ba297d6a73e..f2366adb168 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -60,4 +60,8 @@ extern const uint32_t AES_Te0[256], AES_Te1[256], AES_Te2[256], extern const uint32_t AES_Td0[256], AES_Td1[256], AES_Td2[256], AES_Td3[256], AES_Td4[256]; +/* Pre-byte-swapped, interleaved encryption tables for LATX AES IR2. */ +extern uint32_t AES_Te_latx[256][4]; +extern uint32_t AES_sbox_latx[256][4]; + #endif diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index a6e5d34ff1e..31e7249cf0a 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -233,8 +233,8 @@ typedef enum aot_rel_kind { LOAD_HELPER_AESKEYGENASSIST_XMM, LOAD_HELPER_AESDEC_XMM, LOAD_HELPER_AESDECLAST_XMM, - LOAD_HOST_AES_TE0, - LOAD_HOST_AES_SBOX, + LOAD_HOST_AES_TE_LATX, + LOAD_HOST_AES_SBOX_LATX, LOAD_HELPER_SHA1NEXTE, LOAD_HELPER_SHA1MSG1, LOAD_HELPER_SHA1MSG2, diff --git a/target/i386/latx/include/env.h b/target/i386/latx/include/env.h index 7ff6c0b7cec..630fa016282 100644 --- a/target/i386/latx/include/env.h +++ b/target/i386/latx/include/env.h @@ -64,6 +64,10 @@ typedef struct TRANSLATION_DATA { bool softfpu_region_active; + /* Read-only AES table address retained across consecutive AES rounds. */ + int aes_table_reg; + uint8_t aes_table_kind; + /* manage the immediate number cache */ IMM_CACHE *imm_cache; } TRANSLATION_DATA; diff --git a/target/i386/latx/sbt/aot.c b/target/i386/latx/sbt/aot.c index 7fec55fb7e5..1f287f7fe87 100644 --- a/target/i386/latx/sbt/aot.c +++ b/target/i386/latx/sbt/aot.c @@ -1267,8 +1267,8 @@ static void* relkind_to_fixup_addr[] = { [LOAD_HELPER_AESKEYGENASSIST_XMM] = helper_aeskeygenassist_xmm, [LOAD_HELPER_AESDEC_XMM] = helper_aesdec_xmm, [LOAD_HELPER_AESDECLAST_XMM] = helper_aesdeclast_xmm, - [LOAD_HOST_AES_TE0] = (void *)AES_Te0, - [LOAD_HOST_AES_SBOX] = (void *)AES_sbox, + [LOAD_HOST_AES_TE_LATX] = (void *)AES_Te_latx, + [LOAD_HOST_AES_SBOX_LATX] = (void *)AES_sbox_latx, [LOAD_HELPER_SHA1NEXTE] = helper_sha1nexte, [LOAD_HELPER_SHA1MSG1] = helper_sha1msg1, [LOAD_HELPER_SHA1MSG2] = helper_sha1msg2, diff --git a/target/i386/latx/translator/tr-simd.c b/target/i386/latx/translator/tr-simd.c index 8c39c75c5fa..d01ad7f6257 100644 --- a/target/i386/latx/translator/tr-simd.c +++ b/target/i386/latx/translator/tr-simd.c @@ -3770,48 +3770,60 @@ bool translate_pclmulqdq(IR1_INST * pir1) { static void aes_load_te0(IR2_OPND value, IR2_OPND index, IR2_OPND table, IR2_OPND state, - int byte_shift, int rotate) + int byte_shift, int column) { la_bstrpick_d(index, state, byte_shift + 7, byte_shift); - /* IR2 ALSL encodes shift-1, so 1 means index * sizeof(uint32_t). */ - la_alsl_d(index, index, table, 1); - la_ld_wu(value, index, 0); - if (rotate) { - la_rotri_w(value, value, rotate); + /* IR2 ALSL encodes shift-1, so 3 means a 16-byte table row. */ + la_alsl_d(index, index, table, 3); + la_ld_wu(value, index, column * sizeof(uint32_t)); +} + +static void aes_load_table_once(IR2_OPND table, ADDR address, + enum aot_rel_kind rel_kind, uint8_t kind) +{ + TRANSLATION_DATA *t = lsenv->tr_data; + int reg = ir2_opnd_base_reg_num(&table); + + if (t->aes_table_kind != kind || t->aes_table_reg != reg) { + aot_load_host_addr(table, address, rel_kind, 0); + t->aes_table_reg = reg; + t->aes_table_kind = kind; } } static void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) { - IR2_OPND state[4]; + IR2_OPND state[2]; IR2_OPND table = ra_alloc_itemp(); IR2_OPND index = ra_alloc_itemp(); IR2_OPND result = ra_alloc_itemp(); - for (int i = 0; i < 4; i++) { + for (int i = 0; i < 2; i++) { state[i] = ra_alloc_itemp(); - la_vpickve2gr_wu(state[i], dest, i); + la_vpickve2gr_du(state[i], dest, i); } - aot_load_host_addr(table, (ADDR)AES_Te0, LOAD_HOST_AES_TE0, 0); + aes_load_table_once(table, (ADDR)AES_Te_latx, + LOAD_HOST_AES_TE_LATX, 1); for (int word = 0; word < 4; word++) { - aes_load_te0(result, index, table, state[word], 0, 0); + int src_word = word; + + aes_load_te0(result, index, table, state[src_word >> 1], + (src_word & 1) * 32, 0); for (int column = 1; column < 4; column++) { + src_word = (word + column) & 3; aes_load_te0(index, index, table, - state[(word + column) & 3], column * 8, - column * 8); + state[src_word >> 1], + (src_word & 1) * 32 + column * 8, + column); la_xor(result, result, index); } - /* bswap32 before XORing the round key, matching AESENC semantics. */ - la_revb_2h(result, result); - la_rotri_w(result, result, 16); - la_vpickve2gr_wu(index, key, word); - la_xor(result, result, index); la_vinsgr2vr_w(dest, result, word); } + la_vxor_v(dest, dest, key); - for (int i = 0; i < 4; i++) { + for (int i = 0; i < 2; i++) { ra_free_temp(state[i]); } ra_free_temp(table); @@ -3821,41 +3833,45 @@ static void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) static void aes_load_sbox(IR2_OPND value, IR2_OPND index, IR2_OPND table, IR2_OPND state, - int byte_shift) + int byte_shift, int column) { la_bstrpick_d(index, state, byte_shift + 7, byte_shift); - la_add_d(index, index, table); - la_ld_bu(value, index, 0); + la_alsl_d(index, index, table, 3); + la_ld_wu(value, index, column * sizeof(uint32_t)); } static void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key) { - IR2_OPND state[4]; + IR2_OPND state[2]; IR2_OPND table = ra_alloc_itemp(); IR2_OPND index = ra_alloc_itemp(); IR2_OPND result = ra_alloc_itemp(); - for (int i = 0; i < 4; i++) { + for (int i = 0; i < 2; i++) { state[i] = ra_alloc_itemp(); - la_vpickve2gr_wu(state[i], dest, i); + la_vpickve2gr_du(state[i], dest, i); } - aot_load_host_addr(table, (ADDR)AES_sbox, LOAD_HOST_AES_SBOX, 0); + aes_load_table_once(table, (ADDR)AES_sbox_latx, + LOAD_HOST_AES_SBOX_LATX, 2); for (int word = 0; word < 4; word++) { - aes_load_sbox(result, index, table, state[word], 0); + int src_word = word; + + aes_load_sbox(result, index, table, state[src_word >> 1], + (src_word & 1) * 32, 0); for (int byte = 1; byte < 4; byte++) { + src_word = (word + byte) & 3; aes_load_sbox(index, index, table, - state[(word + byte) & 3], byte * 8); - la_slli_w(index, index, byte * 8); + state[src_word >> 1], + (src_word & 1) * 32 + byte * 8, byte); la_or(result, result, index); } - la_vpickve2gr_wu(index, key, word); - la_xor(result, result, index); la_vinsgr2vr_w(dest, result, word); } + la_vxor_v(dest, dest, key); - for (int i = 0; i < 4; i++) { + for (int i = 0; i < 2; i++) { ra_free_temp(state[i]); } ra_free_temp(table); @@ -3949,6 +3965,8 @@ static void load_aes_key_from_mem(IR2_OPND key, IR1_OPND *mem) la_vld(key, address, offset); /* AES needs every x86-64 itemp; the emitted load no longer needs this. */ ra_free_temp_auto(address); + /* convert_mem may reuse the GPR holding the cached AES table address. */ + lsenv->tr_data->aes_table_kind = 0; } bool translate_aesenc(IR1_INST *pir1) diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 38266d7bbcb..d937672e65a 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -139,6 +139,7 @@ void tr_init(void *tb) t->imm_cache->bucket = (IMM_CACHE_BUCKET *)mm_calloc( CACHE_MAX_CAPACITY, sizeof(IMM_CACHE_BUCKET)); } + t->aes_table_kind = 0; } void tr_fini(bool check_the_extension) @@ -1865,6 +1866,16 @@ static bool (*translate_functions[])(IR1_INST *) = { TRANS_FUNC_GEN_REAL(ENDING, NULL), }; +static void reset_aes_table_cache_for_non_aes(IR1_INST *ir1) +{ + TRANSLATION_DATA *t = lsenv->tr_data; + + if (ir1_opcode(ir1) != dt_X86_INS_AESENC && + ir1_opcode(ir1) != dt_X86_INS_AESENCLAST) { + t->aes_table_kind = 0; + } +} + bool ir1_translate(IR1_INST *ir1) { #ifdef CONFIG_LATX_INSTS_PATTERN @@ -1885,6 +1896,8 @@ bool ir1_translate(IR1_INST *ir1) } #endif + reset_aes_table_cache_for_non_aes(ir1); + if (ir1_opcode(ir1) == dt_X86_INS_CALL) { if (!ir1_is_indirect_call(ir1)) { if (ir1_addr_next(ir1) == ir1_target_addr(ir1)) { From 59bf805fedbc7f99bad7e77651a2293c501f6aa0 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 26 Aug 2026 11:03:09 +0800 Subject: [PATCH 4/6] LATX, opt: translate VEX AES rounds directly Signed-off-by: Lu Zeng --- target/i386/latx/include/translate.h | 4 ++ target/i386/latx/translator/tr-avx.c | 74 ++++++++++++++++++++----- target/i386/latx/translator/tr-simd.c | 6 +- target/i386/latx/translator/translate.c | 7 ++- 4 files changed, 73 insertions(+), 18 deletions(-) diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 38200bc4298..f519b26c4a0 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1447,6 +1447,10 @@ TRANS_FUNC_DEF(aesenclast); TRANS_FUNC_DEF(aesimc); TRANS_FUNC_DEF(aeskeygenassist); +void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key); +void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key); +void load_aes_key_from_mem(IR2_OPND key, IR1_OPND *mem); + #ifdef CONFIG_LATX_AVX_OPT TRANS_FUNC_DEF(vaesdec); TRANS_FUNC_DEF(vaesdeclast); diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 7c809154e59..33708390d42 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -5900,6 +5900,54 @@ bool translate_vaesdeclast(IR1_INST *pir1) return true; } +static bool translate_vaes_xmm_ir2(IR1_INST *pir1, bool is_last) +{ + IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + IR1_OPND *opnd2 = ir1_get_opnd(pir1, 2); + int d = ir1_opnd_base_reg_num(opnd0); + int s1 = ir1_opnd_base_reg_num(opnd1); + IR2_OPND dest = ra_alloc_xmm(d); + IR2_OPND src1 = ra_alloc_xmm(s1); + IR2_OPND key; + bool key_is_temp = false; + + lsassert(ir1_opnd_is_xmm(opnd0)); + lsassert(ir1_opnd_is_xmm(opnd1)); + + if (ir1_opnd_is_mem(opnd2)) { + key = ra_alloc_ftemp(); + key_is_temp = true; + load_aes_key_from_mem(key, opnd2); + } else { + int s2 = ir1_opnd_base_reg_num(opnd2); + IR2_OPND src2 = ra_alloc_xmm(s2); + + if (s2 == d) { + key = ra_alloc_ftemp(); + key_is_temp = true; + la_vor_v(key, src2, src2); + } else { + key = src2; + } + } + + if (d != s1) { + la_vor_v(dest, src1, src1); + } + if (is_last) { + gen_aesenclast_ir2(dest, key); + } else { + gen_aesenc_ir2(dest, key); + } + set_high128_xreg_to_zero(dest); + + if (key_is_temp) { + ra_free_temp(key); + } + return true; +} + bool translate_vaesenc(IR1_INST *pir1) { if (!option_enable_lasx) { @@ -5916,16 +5964,15 @@ bool translate_vaesenc(IR1_INST *pir1) int d = ir1_opnd_base_reg_num(opnd0); int s1 = ir1_opnd_base_reg_num(opnd1); + if (!ir1_opnd_is_ymm(opnd0)) { + return translate_vaes_xmm_ir2(pir1, false); + } + ADDR helper_func; int helper_kind; - if (ir1_opnd_is_ymm(opnd0)) { - helper_func = (ADDR)helper_vaesenc_ymm; - helper_kind = LOAD_HELPER_VAESENC_YMM; - } else { - helper_func = (ADDR)helper_vaesenc_xmm; - helper_kind = LOAD_HELPER_VAESENC_XMM; - } + helper_func = (ADDR)helper_vaesenc_ymm; + helper_kind = LOAD_HELPER_VAESENC_YMM; if (!ir1_opnd_is_mem(opnd2)) { int s2 = ir1_opnd_base_reg_num(opnd2); @@ -5974,16 +6021,15 @@ bool translate_vaesenclast(IR1_INST *pir1) int d = ir1_opnd_base_reg_num(opnd0); int s1 = ir1_opnd_base_reg_num(opnd1); + if (!ir1_opnd_is_ymm(opnd0)) { + return translate_vaes_xmm_ir2(pir1, true); + } + ADDR helper_func; int helper_kind; - if (ir1_opnd_is_ymm(opnd0)) { - helper_func = (ADDR)helper_vaesenclast_ymm; - helper_kind = LOAD_HELPER_VAESENCLAST_YMM; - } else { - helper_func = (ADDR)helper_vaesenclast_xmm; - helper_kind = LOAD_HELPER_VAESENCLAST_XMM; - } + helper_func = (ADDR)helper_vaesenclast_ymm; + helper_kind = LOAD_HELPER_VAESENCLAST_YMM; if (!ir1_opnd_is_mem(opnd2)) { int s2 = ir1_opnd_base_reg_num(opnd2); diff --git a/target/i386/latx/translator/tr-simd.c b/target/i386/latx/translator/tr-simd.c index d01ad7f6257..602b7d804f6 100644 --- a/target/i386/latx/translator/tr-simd.c +++ b/target/i386/latx/translator/tr-simd.c @@ -3791,7 +3791,7 @@ static void aes_load_table_once(IR2_OPND table, ADDR address, } } -static void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) +void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) { IR2_OPND state[2]; IR2_OPND table = ra_alloc_itemp(); @@ -3840,7 +3840,7 @@ static void aes_load_sbox(IR2_OPND value, IR2_OPND index, la_ld_wu(value, index, column * sizeof(uint32_t)); } -static void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key) +void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key) { IR2_OPND state[2]; IR2_OPND table = ra_alloc_itemp(); @@ -3953,7 +3953,7 @@ bool translate_aesdeclast(IR1_INST *pir1) return true; } -static void load_aes_key_from_mem(IR2_OPND key, IR1_OPND *mem) +void load_aes_key_from_mem(IR2_OPND key, IR1_OPND *mem) { int offset; IR2_OPND address; diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index d937672e65a..92c6a04fc2b 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -1871,7 +1871,12 @@ static void reset_aes_table_cache_for_non_aes(IR1_INST *ir1) TRANSLATION_DATA *t = lsenv->tr_data; if (ir1_opcode(ir1) != dt_X86_INS_AESENC && - ir1_opcode(ir1) != dt_X86_INS_AESENCLAST) { + ir1_opcode(ir1) != dt_X86_INS_AESENCLAST +#ifdef CONFIG_LATX_AVX_OPT + && ir1_opcode(ir1) != dt_X86_INS_VAESENC + && ir1_opcode(ir1) != dt_X86_INS_VAESENCLAST +#endif + ) { t->aes_table_kind = 0; } } From d11ed178d38c85ed5d5c89022b7497411df9869c Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 26 Aug 2026 11:25:07 +0800 Subject: [PATCH 5/6] LATX, opt: translate VAES YMM rounds with LASX Signed-off-by: Lu Zeng --- crypto/aes.c | 19 ++++ include/crypto/aes.h | 1 + linux-user/syscall.c | 8 ++ target/i386/cpu.c | 6 +- target/i386/latx/include/aot.h | 1 + target/i386/latx/include/translate.h | 3 + target/i386/latx/sbt/aot.c | 1 + target/i386/latx/translator/tr-avx.c | 120 +++++++++++-------------- target/i386/latx/translator/tr-simd.c | 125 ++++++++++++++++++++++++-- 9 files changed, 207 insertions(+), 77 deletions(-) diff --git a/crypto/aes.c b/crypto/aes.c index 78d837952f3..7e4174f1ab3 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -665,6 +665,25 @@ const uint32_t AES_Te3[256] = { uint32_t AES_Te_latx[256][4]; uint32_t AES_sbox_latx[256][4]; +#define AES_VPAES_DUP16(...) { __VA_ARGS__, __VA_ARGS__ } +const uint8_t AES_vpaes_lasx[7][32] __attribute__((aligned(32))) = { + AES_VPAES_DUP16(0x10,0x01,0x08,0x0d,0x0f,0x06,0x05,0x0e, + 0x02,0x0c,0x0b,0x0a,0x09,0x03,0x07,0x04), + AES_VPAES_DUP16(0x10,0x07,0x0b,0x0f,0x06,0x0a,0x04,0x01, + 0x09,0x08,0x05,0x02,0x0c,0x0e,0x0d,0x03), + AES_VPAES_DUP16(0x00,0x70,0x2a,0x5a,0x98,0xe8,0xb2,0xc2, + 0x08,0x78,0x22,0x52,0x90,0xe0,0xba,0xca), + AES_VPAES_DUP16(0x00,0x4d,0x7c,0x31,0x7d,0x30,0x01,0x4c, + 0x81,0xcc,0xfd,0xb0,0xfc,0xb1,0x80,0xcd), + AES_VPAES_DUP16(0x00,0xc7,0xbd,0x6f,0x17,0x6d,0xd2,0xd0, + 0x78,0xa8,0x02,0xc5,0x7a,0xbf,0xaa,0x15), + AES_VPAES_DUP16(0x00,0x6a,0xbb,0x5f,0xa5,0x74,0xe4,0xcf, + 0xfa,0x35,0x2b,0x41,0xd1,0x90,0x1e,0x8e), + AES_VPAES_DUP16(0x00,0x05,0x0a,0x0f,0x04,0x09,0x0e,0x03, + 0x08,0x0d,0x02,0x07,0x0c,0x01,0x06,0x0b), +}; +#undef AES_VPAES_DUP16 + static void __attribute__((constructor)) aes_init_latx_te(void) { for (int i = 0; i < 256; i++) { diff --git a/include/crypto/aes.h b/include/crypto/aes.h index f2366adb168..691c7a4fcdb 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -63,5 +63,6 @@ extern const uint32_t AES_Td0[256], AES_Td1[256], AES_Td2[256], /* Pre-byte-swapped, interleaved encryption tables for LATX AES IR2. */ extern uint32_t AES_Te_latx[256][4]; extern uint32_t AES_sbox_latx[256][4]; +extern const uint8_t AES_vpaes_lasx[7][32]; #endif diff --git a/linux-user/syscall.c b/linux-user/syscall.c index f9428b333f6..293849551a9 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -12770,6 +12770,7 @@ CPUID_GROUP_EXT, CPUID_GROUP_EXT2, CPUID_GROUP_EXT3, CPUID_GROUP_EBX, +CPUID_GROUP_ECX, CPUID_GROUP_XSAVE, }; const char *__x86_cpuid_flags_table[] = { @@ -12800,6 +12801,11 @@ const char *__x86_cpuid_flags_table[] = { "avx512f", "avx512dq", "rdseed","adx", "smap", "avx512ifma", "pcommit", "clflushopt", "clwb", "intel_pt", "avx512pf", "avx512er", "avx512cd", "sha_ni", "avx512bw", "avx512vl", /*xsave*/ +"", "avx512vbmi", "umip", "pku", "ospke", "waitpkg", "avx512vbmi2", "", +"gfni", "vaes", "vpclmulqdq", "avx512vnni", "avx512bitalg", "", "avx512-vpopcntdq", "", +"la57", "", "", "", "", "", "rdpid", "", +"bus-lock-detect", "cldemote", "", "movdiri", "movdir64b", "", "", "pks", +/*xsave*/ "xsaveopt", "xsavec", "xgetbv1", "xsaves", "", "", "", "", "", "", "", "", "" "", "", "", "", "", "", "", "", "", "", "", @@ -12825,6 +12831,8 @@ static char *cpuinfo_flags(CPUArchState *env) group = CPUID_GROUP_EXT3; } else if (i == FEAT_7_0_EBX) { group = CPUID_GROUP_EBX; + } else if (i == FEAT_7_0_ECX) { + group = CPUID_GROUP_ECX; } else if (i == FEAT_XSAVE) { group = CPUID_GROUP_XSAVE; } else if (i == FEAT_XSAVE_COMP_LO || i ==FEAT_XSAVE_COMP_HI) { diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 57f25697fb8..7efd3f1dd13 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -687,7 +687,7 @@ static void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1, #ifdef CONFIG_LATX_AVX_OPT #define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_PKU | \ /* CPUID_7_0_ECX_OSPKE is dynamic */ \ - CPUID_7_0_ECX_LA57) + CPUID_7_0_ECX_LA57 | CPUID_7_0_ECX_VAES) #else/*CONFIG_LATX_AVX_OPT*/ #define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_PKU | \ /* CPUID_7_0_ECX_OSPKE is dynamic */ \ @@ -1933,6 +1933,8 @@ static X86CPUDefinition builtin_x86_defs[] = { CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_BMI2, + .features[FEAT_7_0_ECX] = + CPUID_7_0_ECX_VAES, .features[FEAT_XSAVE] = CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XGETBV1, #endif/*CONFIG_LATX_AVX_OPT*/ @@ -2081,6 +2083,8 @@ static X86CPUDefinition builtin_x86_defs[] = { CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_BMI2, + .features[FEAT_7_0_ECX] = + CPUID_7_0_ECX_VAES, .features[FEAT_XSAVE] = CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XGETBV1, #endif diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index 31e7249cf0a..56e4c8c4a95 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -235,6 +235,7 @@ typedef enum aot_rel_kind { LOAD_HELPER_AESDECLAST_XMM, LOAD_HOST_AES_TE_LATX, LOAD_HOST_AES_SBOX_LATX, + LOAD_HOST_AES_VPAES_LASX, LOAD_HELPER_SHA1NEXTE, LOAD_HELPER_SHA1MSG1, LOAD_HELPER_SHA1MSG2, diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index f519b26c4a0..e005e2c123d 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1449,6 +1449,9 @@ TRANS_FUNC_DEF(aeskeygenassist); void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key); void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key); +void gen_aesenc_ir2_no_key(IR2_OPND dest); +void gen_aesenclast_ir2_no_key(IR2_OPND dest); +void gen_vaesenc_lasx_ir2(IR2_OPND dest, IR2_OPND key, bool is_last); void load_aes_key_from_mem(IR2_OPND key, IR1_OPND *mem); #ifdef CONFIG_LATX_AVX_OPT diff --git a/target/i386/latx/sbt/aot.c b/target/i386/latx/sbt/aot.c index 1f287f7fe87..b61073c8c36 100644 --- a/target/i386/latx/sbt/aot.c +++ b/target/i386/latx/sbt/aot.c @@ -1269,6 +1269,7 @@ static void* relkind_to_fixup_addr[] = { [LOAD_HELPER_AESDECLAST_XMM] = helper_aesdeclast_xmm, [LOAD_HOST_AES_TE_LATX] = (void *)AES_Te_latx, [LOAD_HOST_AES_SBOX_LATX] = (void *)AES_sbox_latx, + [LOAD_HOST_AES_VPAES_LASX] = (void *)AES_vpaes_lasx, [LOAD_HELPER_SHA1NEXTE] = helper_sha1nexte, [LOAD_HELPER_SHA1MSG1] = helper_sha1msg1, [LOAD_HELPER_SHA1MSG2] = helper_sha1msg2, diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 33708390d42..a61e10cb2c1 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -5964,45 +5964,35 @@ bool translate_vaesenc(IR1_INST *pir1) int d = ir1_opnd_base_reg_num(opnd0); int s1 = ir1_opnd_base_reg_num(opnd1); - if (!ir1_opnd_is_ymm(opnd0)) { - return translate_vaes_xmm_ir2(pir1, false); - } - - ADDR helper_func; - int helper_kind; - - helper_func = (ADDR)helper_vaesenc_ymm; - helper_kind = LOAD_HELPER_VAESENC_YMM; - - if (!ir1_opnd_is_mem(opnd2)) { - int s2 = ir1_opnd_base_reg_num(opnd2); - tr_gen_call_to_helper_aes((ADDR)helper_func, d, s1, s2, - helper_kind); - } else { - int s2 = 0; - while (s2 < 8) { - if (s2 != d && s2 != s1) { - break; + if (ir1_opnd_is_ymm(opnd0)) { + IR2_OPND dest = ra_alloc_xmm(d); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2; + bool src2_is_temp = false; + if (ir1_opnd_is_mem(opnd2)) { + src2 = ra_alloc_ftemp(); + load_freg256_from_ir1_mem(src2, opnd2); + src2_is_temp = true; + } else { + src2 = load_freg256_from_ir1(opnd2); + if (ir1_opnd_base_reg_num(opnd2) == d) { + IR2_OPND key = ra_alloc_ftemp(); + la_xvor_v(key, src2, src2); + src2 = key; + src2_is_temp = true; } - s2++; } - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND src = ra_alloc_xmm(s2); - la_xvor_v(temp, src, src); - if (ir1_opnd_size(opnd2) == 128) { - load_freg128_from_ir1_mem(src, opnd2); - } else { - load_freg256_from_ir1_mem(src, opnd2); + if (d != s1) { + la_xvor_v(dest, src1, src1); } - tr_gen_call_to_helper_aes((ADDR)helper_func, d, s1, s2, - helper_kind); - la_xvor_v(src, temp, temp); - } - if (!ir1_opnd_is_ymm(opnd0)) { - set_high128_xreg_to_zero(ra_alloc_xmm(d)); + gen_vaesenc_lasx_ir2(dest, src2, false); + if (src2_is_temp) { + ra_free_temp(src2); + } + return true; } - /* TODO: need to check */ - return true; + + return translate_vaes_xmm_ir2(pir1, false); } bool translate_vaesenclast(IR1_INST *pir1) @@ -6021,45 +6011,35 @@ bool translate_vaesenclast(IR1_INST *pir1) int d = ir1_opnd_base_reg_num(opnd0); int s1 = ir1_opnd_base_reg_num(opnd1); - if (!ir1_opnd_is_ymm(opnd0)) { - return translate_vaes_xmm_ir2(pir1, true); - } - - ADDR helper_func; - int helper_kind; - - helper_func = (ADDR)helper_vaesenclast_ymm; - helper_kind = LOAD_HELPER_VAESENCLAST_YMM; - - if (!ir1_opnd_is_mem(opnd2)) { - int s2 = ir1_opnd_base_reg_num(opnd2); - tr_gen_call_to_helper_aes((ADDR)helper_func, d, s1, s2, - helper_kind); - } else { - int s2 = 0; - while (s2 < 8) { - if (s2 != d && s2 != s1) { - break; + if (ir1_opnd_is_ymm(opnd0)) { + IR2_OPND dest = ra_alloc_xmm(d); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + IR2_OPND src2; + bool src2_is_temp = false; + if (ir1_opnd_is_mem(opnd2)) { + src2 = ra_alloc_ftemp(); + load_freg256_from_ir1_mem(src2, opnd2); + src2_is_temp = true; + } else { + src2 = load_freg256_from_ir1(opnd2); + if (ir1_opnd_base_reg_num(opnd2) == d) { + IR2_OPND key = ra_alloc_ftemp(); + la_xvor_v(key, src2, src2); + src2 = key; + src2_is_temp = true; } - s2++; } - IR2_OPND temp = ra_alloc_ftemp(); - IR2_OPND src = ra_alloc_xmm(s2); - la_xvor_v(temp, src, src); - if (ir1_opnd_size(opnd2) == 128) { - load_freg128_from_ir1_mem(src, opnd2); - } else { - load_freg256_from_ir1_mem(src, opnd2); + if (d != s1) { + la_xvor_v(dest, src1, src1); } - tr_gen_call_to_helper_aes((ADDR)helper_func, d, s1, s2, - helper_kind); - la_xvor_v(src, temp, temp); - } - if (!ir1_opnd_is_ymm(opnd0)) { - set_high128_xreg_to_zero(ra_alloc_xmm(d)); + gen_vaesenc_lasx_ir2(dest, src2, true); + if (src2_is_temp) { + ra_free_temp(src2); + } + return true; } - /* TODO: need to check */ - return true; + + return translate_vaes_xmm_ir2(pir1, true); } bool translate_vaesimc(IR1_INST *pir1) diff --git a/target/i386/latx/translator/tr-simd.c b/target/i386/latx/translator/tr-simd.c index 602b7d804f6..3f650c8569f 100644 --- a/target/i386/latx/translator/tr-simd.c +++ b/target/i386/latx/translator/tr-simd.c @@ -3791,7 +3791,7 @@ static void aes_load_table_once(IR2_OPND table, ADDR address, } } -void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) +void gen_aesenc_ir2_no_key(IR2_OPND dest) { IR2_OPND state[2]; IR2_OPND table = ra_alloc_itemp(); @@ -3821,8 +3821,6 @@ void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) la_vinsgr2vr_w(dest, result, word); } - la_vxor_v(dest, dest, key); - for (int i = 0; i < 2; i++) { ra_free_temp(state[i]); } @@ -3831,6 +3829,12 @@ void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) ra_free_temp(result); } +void gen_aesenc_ir2(IR2_OPND dest, IR2_OPND key) +{ + gen_aesenc_ir2_no_key(dest); + la_vxor_v(dest, dest, key); +} + static void aes_load_sbox(IR2_OPND value, IR2_OPND index, IR2_OPND table, IR2_OPND state, int byte_shift, int column) @@ -3840,7 +3844,7 @@ static void aes_load_sbox(IR2_OPND value, IR2_OPND index, la_ld_wu(value, index, column * sizeof(uint32_t)); } -void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key) +void gen_aesenclast_ir2_no_key(IR2_OPND dest) { IR2_OPND state[2]; IR2_OPND table = ra_alloc_itemp(); @@ -3869,8 +3873,6 @@ void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key) la_vinsgr2vr_w(dest, result, word); } - la_vxor_v(dest, dest, key); - for (int i = 0; i < 2; i++) { ra_free_temp(state[i]); } @@ -3879,6 +3881,117 @@ void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key) ra_free_temp(result); } +void gen_aesenclast_ir2(IR2_OPND dest, IR2_OPND key) +{ + gen_aesenclast_ir2_no_key(dest); + la_vxor_v(dest, dest, key); +} + +enum { + VPAES_INV, + VPAES_INVA, + VPAES_IPT_LO, + VPAES_IPT_HI, + VPAES_SBO_U, + VPAES_SBO_T, + VPAES_SR1, +}; + +static void vpaes_load_lasx_const(IR2_OPND dest, IR2_OPND base, int index) +{ + la_xvld(dest, base, index * 32); +} + +static void vpaes_lasx_shuf(IR2_OPND dest, IR2_OPND table, + IR2_OPND index, IR2_OPND zero) +{ + la_xvshuf_b(dest, zero, table, index); +} + +void gen_vaesenc_lasx_ir2(IR2_OPND dest, IR2_OPND key, bool is_last) +{ + IR2_OPND base = ra_alloc_itemp(); + IR2_OPND zero = ra_alloc_ftemp(); + IR2_OPND mask = ra_alloc_ftemp(); + IR2_OPND hi = ra_alloc_ftemp(); + IR2_OPND lo = ra_alloc_ftemp(); + IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND aux = ra_alloc_ftemp(); + + aes_load_table_once(base, (ADDR)AES_vpaes_lasx, + LOAD_HOST_AES_VPAES_LASX, 3); + la_xvxor_v(zero, zero, zero); + la_xvldi(mask, 0x0f); + + la_xvsrli_b(hi, dest, 4); + la_xvand_v(lo, dest, mask); + vpaes_load_lasx_const(temp, base, VPAES_IPT_LO); + vpaes_lasx_shuf(temp, temp, lo, zero); + vpaes_load_lasx_const(aux, base, VPAES_IPT_HI); + vpaes_lasx_shuf(aux, aux, hi, zero); + la_xvxor_v(dest, temp, aux); + + la_xvsrli_b(hi, dest, 4); + la_xvand_v(lo, dest, mask); + + vpaes_load_lasx_const(temp, base, VPAES_INVA); + vpaes_lasx_shuf(dest, temp, lo, zero); + la_xvxor_v(mask, lo, hi); + + vpaes_load_lasx_const(temp, base, VPAES_INV); + vpaes_lasx_shuf(temp, temp, hi, zero); + la_xvxor_v(temp, temp, dest); + + vpaes_load_lasx_const(lo, base, VPAES_INV); + vpaes_lasx_shuf(lo, lo, mask, zero); + la_xvxor_v(lo, lo, dest); + + vpaes_load_lasx_const(dest, base, VPAES_INV); + vpaes_lasx_shuf(dest, dest, temp, zero); + la_xvxor_v(dest, dest, mask); + + vpaes_load_lasx_const(temp, base, VPAES_INV); + vpaes_lasx_shuf(temp, temp, lo, zero); + la_xvxor_v(temp, temp, hi); + + vpaes_load_lasx_const(lo, base, VPAES_SBO_U); + vpaes_lasx_shuf(lo, lo, dest, zero); + vpaes_load_lasx_const(mask, base, VPAES_SBO_T); + vpaes_lasx_shuf(mask, mask, temp, zero); + la_xvxor_v(dest, lo, mask); + la_xvxori_b(dest, dest, 0x63); + + vpaes_load_lasx_const(temp, base, VPAES_SR1); + vpaes_lasx_shuf(dest, dest, temp, zero); + + if (!is_last) { + la_xvshuf4i_b(mask, dest, 0x39); + la_xvshuf4i_b(hi, dest, 0x4e); + la_xvshuf4i_b(lo, dest, 0x93); + + la_xvxor_v(temp, dest, mask); + la_xvxor_v(temp, temp, hi); + la_xvxor_v(temp, temp, lo); + la_xvxor_v(mask, dest, mask); + la_xvslli_b(hi, mask, 1); + la_xvsrai_b(lo, mask, 7); + la_xvldi(mask, 0x1b); + la_xvand_v(lo, lo, mask); + la_xvxor_v(hi, hi, lo); + la_xvxor_v(dest, dest, temp); + la_xvxor_v(dest, dest, hi); + } + la_xvxor_v(dest, dest, key); + + ra_free_temp(base); + ra_free_temp(zero); + ra_free_temp(mask); + ra_free_temp(hi); + ra_free_temp(lo); + ra_free_temp(temp); + ra_free_temp(aux); +} + bool translate_aesdec(IR1_INST *pir1) { if (option_vpaes) { From 323d87efb44281cf17f128ffdbc18265b9ae4a67 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 26 Aug 2026 15:37:14 +0800 Subject: [PATCH 6/6] LATX, tests: cover AES and VAES JIT and AOT Signed-off-by: Lu Zeng --- tests/integration/aes-rounds.S | 114 ++++++++++++++++ tests/integration/aesenc-all-bytes.S | 64 +++++++++ tests/integration/meson.build | 1 + .../integration/registrations/aes/meson.build | 14 ++ tests/integration/test-aes-vaes-rounds.sh | 48 +++++++ tests/integration/vaes-xmm-rounds.S | 122 +++++++++++++++++ tests/integration/vaes-ymm-rounds.S | 124 ++++++++++++++++++ 7 files changed, 487 insertions(+) create mode 100644 tests/integration/aes-rounds.S create mode 100644 tests/integration/aesenc-all-bytes.S create mode 100644 tests/integration/registrations/aes/meson.build create mode 100755 tests/integration/test-aes-vaes-rounds.sh create mode 100644 tests/integration/vaes-xmm-rounds.S create mode 100644 tests/integration/vaes-ymm-rounds.S diff --git a/tests/integration/aes-rounds.S b/tests/integration/aes-rounds.S new file mode 100644 index 00000000000..4267882d1d1 --- /dev/null +++ b/tests/integration/aes-rounds.S @@ -0,0 +1,114 @@ +.intel_syntax noprefix +.global _start + +.macro CHECK_XMM reg, expected + movdqu XMMWORD PTR [rip + actual], \reg + mov rax, QWORD PTR [rip + actual] + cmp rax, QWORD PTR [rip + \expected] + jne .Lcheck_fail\@ + mov rax, QWORD PTR [rip + actual + 8] + cmp rax, QWORD PTR [rip + \expected + 8] + je .Lcheck_done\@ +.Lcheck_fail\@: + inc DWORD PTR [rip + fail_count] +.Lcheck_done\@: +.endm + +.section .data +fail_count: .long 0 + +.section .bss +.balign 16 +actual: .zero 16 + +.section .rodata +.balign 16 +state: .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f +key: .byte 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f +memkey: .byte 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f + +aesenc_reg: .byte 0x7a,0x7b,0x4e,0x56,0x38,0x78,0x25,0x46,0xa8,0xc0,0x47,0x7a,0x3b,0x81,0x3f,0x43 +aesenclast_reg: .byte 0x73,0x7a,0x75,0x65,0xe6,0x14,0xbd,0x6c,0x28,0xce,0x6d,0xde,0xe2,0x61,0x71,0x34 +aesenc_mem: .byte 0x4a,0x4b,0x7e,0x66,0x08,0x48,0x15,0x76,0x98,0xf0,0x77,0x4a,0x0b,0xb1,0x0f,0x73 +aesenclast_mem: .byte 0x43,0x4a,0x45,0x55,0xd6,0x24,0x8d,0x5c,0x18,0xfe,0x5d,0xee,0xd2,0x51,0x41,0x04 +aesenc_alias: .byte 0x6a,0x6b,0x5e,0x46,0x28,0x68,0x35,0x56,0xb8,0xd0,0x57,0x6a,0x2b,0x91,0x2f,0x53 +aesdec_reg: .byte 0x03,0xb2,0x84,0x0f,0x3a,0x0c,0x3f,0x79,0x0f,0x56,0xe8,0xa9,0xda,0xa0,0x85,0x4e +aesdeclast_reg: .byte 0x42,0xe2,0xb1,0x2b,0x24,0x1c,0xc1,0x89,0xa7,0x2f,0x70,0xe0,0x9d,0x5d,0xbb,0xca +aesdec_mem: .byte 0x33,0x82,0xb4,0x3f,0x0a,0x3c,0x0f,0x49,0x3f,0x66,0xd8,0x99,0xea,0x90,0xb5,0x7e +aesdeclast_mem: .byte 0x72,0xd2,0x81,0x1b,0x14,0x2c,0xf1,0xb9,0x97,0x1f,0x40,0xd0,0xad,0x6d,0x8b,0xfa +aes_chain: .byte 0x3f,0x53,0x96,0x57,0xb6,0x67,0xe6,0xff,0x39,0x7c,0xdb,0xb1,0x11,0xd2,0x0e,0xb6 +aesenc_mem_twice: .byte 0x1b,0xd8,0xdd,0xe0,0x8e,0xbf,0x23,0xeb,0x2c,0xd2,0x11,0xaa,0x3b,0x46,0x65,0x0f +aesenclast_mem_twice: .byte 0x3a,0x17,0x6e,0xd1,0xd2,0x9e,0xa5,0xdb,0x85,0xf8,0x44,0x61,0x99,0xfb,0x73,0x07 + +.section .text +_start: + movdqu xmm1, XMMWORD PTR [rip + key] + + movdqu xmm0, XMMWORD PTR [rip + state] + aesenc xmm0, xmm1 + CHECK_XMM xmm0, aesenc_reg + + movdqu xmm0, XMMWORD PTR [rip + state] + aesenclast xmm0, xmm1 + CHECK_XMM xmm0, aesenclast_reg + + movdqu xmm0, XMMWORD PTR [rip + state] + aesenc xmm0, XMMWORD PTR [rip + memkey] + CHECK_XMM xmm0, aesenc_mem + + movdqu xmm0, XMMWORD PTR [rip + state] + aesenclast xmm0, XMMWORD PTR [rip + memkey] + CHECK_XMM xmm0, aesenclast_mem + + movdqu xmm0, XMMWORD PTR [rip + state] + aesenc xmm0, xmm0 + CHECK_XMM xmm0, aesenc_alias + + movdqu xmm0, XMMWORD PTR [rip + state] + aesdec xmm0, xmm1 + CHECK_XMM xmm0, aesdec_reg + + movdqu xmm0, XMMWORD PTR [rip + state] + aesdeclast xmm0, xmm1 + CHECK_XMM xmm0, aesdeclast_reg + + movdqu xmm0, XMMWORD PTR [rip + state] + aesdec xmm0, XMMWORD PTR [rip + memkey] + CHECK_XMM xmm0, aesdec_mem + + movdqu xmm0, XMMWORD PTR [rip + state] + aesdeclast xmm0, XMMWORD PTR [rip + memkey] + CHECK_XMM xmm0, aesdeclast_mem + + lea rsi, [rip + memkey] + xor edx, edx + movdqu xmm0, XMMWORD PTR [rip + state] + aesenc xmm0, XMMWORD PTR [rsi + rdx * 2] + aesenc xmm0, XMMWORD PTR [rsi + rdx * 2] + CHECK_XMM xmm0, aesenc_mem_twice + + lea rsi, [rip + memkey] + movdqu xmm0, XMMWORD PTR [rip + state] + aesenclast xmm0, XMMWORD PTR [rsi + rdx * 2] + aesenclast xmm0, XMMWORD PTR [rsi + rdx * 2] + CHECK_XMM xmm0, aesenclast_mem_twice + + movdqu xmm0, XMMWORD PTR [rip + state] + .rept 9 + aesenc xmm0, xmm1 + .endr + aesenclast xmm0, xmm1 + CHECK_XMM xmm0, aes_chain + + movdqu xmm0, XMMWORD PTR [rip + state] + mov ecx, 500000 +.Laot_loop: + aesenc xmm0, xmm1 + dec ecx + jne .Laot_loop + + mov edi, DWORD PTR [rip + fail_count] + mov eax, 60 + syscall + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/aesenc-all-bytes.S b/tests/integration/aesenc-all-bytes.S new file mode 100644 index 00000000000..ff981f3a7a3 --- /dev/null +++ b/tests/integration/aesenc-all-bytes.S @@ -0,0 +1,64 @@ +.intel_syntax noprefix +.global _start + +.section .data +fail_count: .long 0 + +.section .bss +.balign 16 +actual: .zero 256 + +.section .rodata +.balign 16 +states: +.set byte_value, 0 +.rept 256 + .byte byte_value + .set byte_value, byte_value + 1 +.endr +expected: + .byte 0x6a,0x6a,0x5c,0x45,0x2c,0x6d,0x33,0x51,0xb0,0xd9,0x5d,0x61,0x27,0x9c,0x21,0x5c + .byte 0x06,0x45,0x97,0x25,0x87,0xa2,0x4d,0x49,0x8f,0x4e,0x8b,0x7a,0x56,0xe5,0x7a,0x3f + .byte 0xc4,0xe8,0x66,0x32,0x8f,0x12,0x9b,0x82,0x44,0xfd,0x9e,0x94,0xf8,0x63,0x71,0x60 + .byte 0x5c,0xdd,0x16,0xf0,0x77,0x32,0x2b,0x15,0xde,0xb6,0xd3,0x22,0x78,0x93,0x1b,0x3b + .byte 0xf2,0x30,0x47,0xb0,0x4e,0x06,0x50,0x0d,0x16,0x5b,0x12,0x62,0x25,0x69,0xd0,0xdf + .byte 0xc8,0xa6,0x82,0x32,0xb3,0xa8,0x77,0x32,0x5b,0xa9,0xcb,0x44,0x74,0x02,0xa9,0xcc + .byte 0xc6,0xe4,0x7a,0x6f,0xf2,0xeb,0x89,0x4e,0xe1,0x5d,0xa2,0x48,0xc6,0xbf,0x58,0xd2 + .byte 0x16,0xd7,0x0e,0x0b,0x82,0x64,0x53,0xed,0xcc,0x6c,0xc7,0x91,0xc7,0x24,0xa0,0xe9 + .byte 0x2e,0x09,0x33,0x43,0xb9,0xcd,0xe5,0x9c,0x70,0x5c,0x86,0x37,0xa5,0x8d,0xa7,0x9e + .byte 0xdd,0x3c,0x57,0x9f,0xba,0x24,0xa5,0x20,0xa9,0xa3,0x05,0xd0,0xbb,0x78,0x58,0x40 + .byte 0x04,0x7a,0x2e,0x63,0x12,0xc9,0x57,0xf8,0x5d,0xe1,0xc7,0x4a,0x29,0xfb,0x4d,0x7a + .byte 0x4d,0x59,0xd9,0x03,0x38,0xa5,0x2b,0xae,0xc8,0x68,0x98,0xb0,0x2d,0xd6,0x14,0xe6 + .byte 0x60,0xfb,0x71,0x08,0xe1,0x15,0xbe,0x2e,0xf4,0x20,0x4e,0x2c,0xb5,0x63,0x61,0x2f + .byte 0x2c,0x11,0x64,0xe3,0xb4,0x63,0xed,0x3c,0x21,0x32,0xc3,0xcb,0x1a,0x42,0x9f,0x30 + .byte 0xf1,0x05,0x57,0xc3,0xc9,0x3c,0x14,0xaf,0xde,0x16,0x42,0x48,0xf3,0x45,0x11,0xf6 + .byte 0x09,0x3a,0x0a,0x68,0x63,0x4d,0x5c,0xe2,0x9f,0x01,0xa4,0xce,0xce,0x20,0x84,0x36 + +.section .text +_start: + pxor xmm1, xmm1 + lea rsi, [rip + states] + lea rdi, [rip + actual] + mov ecx, 16 +.Laes_loop: + movdqu xmm0, XMMWORD PTR [rsi] + aesenc xmm0, xmm1 + movdqu XMMWORD PTR [rdi], xmm0 + add rsi, 16 + add rdi, 16 + dec ecx + jne .Laes_loop + + lea rsi, [rip + actual] + lea rdi, [rip + expected] + mov ecx, 32 + cld + repe cmpsq + je .Lexit + inc DWORD PTR [rip + fail_count] +.Lexit: + mov edi, DWORD PTR [rip + fail_count] + mov eax, 60 + syscall + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/meson.build b/tests/integration/meson.build index be475972688..764a3dfd5ad 100644 --- a/tests/integration/meson.build +++ b/tests/integration/meson.build @@ -4,6 +4,7 @@ if host_machine.cpu_family() == 'loongarch64' subdir('registrations/x11-kzt') subdir('registrations/sandbox') subdir('registrations/process') + subdir('registrations/aes') subdir('registrations/namespace') subdir('registrations/process-vm') subdir('registrations/prctl') diff --git a/tests/integration/registrations/aes/meson.build b/tests/integration/registrations/aes/meson.build new file mode 100644 index 00000000000..44b4bf7c2e8 --- /dev/null +++ b/tests/integration/registrations/aes/meson.build @@ -0,0 +1,14 @@ +if 'x86_64-linux-user' in target_dirs + latx_integration_tests += [{ + 'name': 'test-aes-vaes-rounds', + 'runner': find_program('../../test-aes-vaes-rounds.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../aes-rounds.S'), + files('../../aesenc-all-bytes.S'), + files('../../vaes-xmm-rounds.S'), + files('../../vaes-ymm-rounds.S'), + ], + 'timeout': 120, + }] +endif diff --git a/tests/integration/test-aes-vaes-rounds.sh b/tests/integration/test-aes-vaes-rounds.sh new file mode 100755 index 00000000000..7c460e14121 --- /dev/null +++ b/tests/integration/test-aes-vaes-rounds.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu + +emulator=$1 +shift +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guests" + exit 77 +fi + +for source_file in "$@"; do + name=$(basename "$source_file" .S) + "$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static -Wl,--build-id=none "$source_file" -o "$workdir/$name" + LATX_AOT=0 "$emulator" "$workdir/$name" +done + +run_aot() +{ + name=$1 + aot_home="$workdir/aot-$name" + mkdir -p "$aot_home" + HOME="$aot_home" LATX_AOT=1 "$emulator" "$workdir/$name" + + aot_file= + for _ in $(seq 1 100); do + aot_file=$(find "$aot_home/.cache/latx" -type f -name '*.aot2' -size +0c -print -quit 2>/dev/null || true) + [ -n "$aot_file" ] && break + sleep 0.1 + done + if [ -z "$aot_file" ]; then + echo "FAIL: no non-empty AOT file generated for $name" >&2 + exit 1 + fi + + HOME="$aot_home" LATX_AOT=1 "$emulator" "$workdir/$name" +} + +run_aot aes-rounds +run_aot vaes-xmm-rounds +run_aot vaes-ymm-rounds +echo "PASS: AES, VEX AES, and VAES YMM JIT/cold-AOT/hot-AOT" diff --git a/tests/integration/vaes-xmm-rounds.S b/tests/integration/vaes-xmm-rounds.S new file mode 100644 index 00000000000..4fd0daf1178 --- /dev/null +++ b/tests/integration/vaes-xmm-rounds.S @@ -0,0 +1,122 @@ +.intel_syntax noprefix +.global _start + +.macro CHECK_XMM reg, expected + vmovdqu XMMWORD PTR [rip + actual], \reg + mov rax, QWORD PTR [rip + actual] + cmp rax, QWORD PTR [rip + \expected] + jne .Lcheck_fail\@ + mov rax, QWORD PTR [rip + actual + 8] + cmp rax, QWORD PTR [rip + \expected + 8] + je .Lcheck_done\@ +.Lcheck_fail\@: + inc DWORD PTR [rip + fail_count] +.Lcheck_done\@: +.endm + +.section .data +fail_count: .long 0 + +.section .bss +.balign 32 +actual: .zero 16 + +.section .rodata +.balign 32 +state0: .byte 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 +state1: .byte 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +key: .byte 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +memkey: .byte 32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47 +ones: .fill 16,1,0xff + +enc0: .byte 0x7a,0x7b,0x4e,0x56,0x38,0x78,0x25,0x46,0xa8,0xc0,0x47,0x7a,0x3b,0x81,0x3f,0x43 +enc1: .byte 0x16,0x54,0x85,0x36,0x93,0xb7,0x5b,0x5e,0x97,0x57,0x91,0x61,0x4a,0xf8,0x64,0x20 +last0: .byte 0x73,0x7a,0x75,0x65,0xe6,0x14,0xbd,0x6c,0x28,0xce,0x6d,0xde,0xe2,0x61,0x71,0x34 +last1: .byte 0xda,0x48,0xb0,0xd3,0xee,0xc1,0x64,0x6a,0xb5,0xbd,0xd3,0xeb,0x80,0x9f,0x59,0xb0 +enc_mem: .byte 0x4a,0x4b,0x7e,0x66,0x08,0x48,0x15,0x76,0x98,0xf0,0x77,0x4a,0x0b,0xb1,0x0f,0x73 +last_mem: .byte 0x43,0x4a,0x45,0x55,0xd6,0x24,0x8d,0x5c,0x18,0xfe,0x5d,0xee,0xd2,0x51,0x41,0x04 +chain: .byte 0x3f,0x53,0x96,0x57,0xb6,0x67,0xe6,0xff,0x39,0x7c,0xdb,0xb1,0x11,0xd2,0x0e,0xb6 +enc_mem_twice: .byte 0x1b,0xd8,0xdd,0xe0,0x8e,0xbf,0x23,0xeb,0x2c,0xd2,0x11,0xaa,0x3b,0x46,0x65,0x0f +last_mem_twice: .byte 0x3a,0x17,0x6e,0xd1,0xd2,0x9e,0xa5,0xdb,0x85,0xf8,0x44,0x61,0x99,0xfb,0x73,0x07 + +.section .text +_start: + vmovdqu xmm1, XMMWORD PTR [rip + state0] + vmovdqu xmm2, XMMWORD PTR [rip + key] + + vaesenc xmm0, xmm1, xmm2 + CHECK_XMM xmm0, enc0 + vaesenclast xmm0, xmm1, xmm2 + CHECK_XMM xmm0, last0 + vaesenc xmm0, xmm1, XMMWORD PTR [rip + memkey] + CHECK_XMM xmm0, enc_mem + vaesenclast xmm0, xmm1, XMMWORD PTR [rip + memkey] + CHECK_XMM xmm0, last_mem + + lea rsi, [rip + memkey] + xor edx, edx + vmovdqu xmm0, XMMWORD PTR [rip + state0] + vaesenc xmm0, xmm0, XMMWORD PTR [rsi + rdx * 2] + vaesenc xmm0, xmm0, XMMWORD PTR [rsi + rdx * 2] + CHECK_XMM xmm0, enc_mem_twice + + lea rsi, [rip + memkey] + vmovdqu xmm0, XMMWORD PTR [rip + state0] + vaesenclast xmm0, xmm0, XMMWORD PTR [rsi + rdx * 2] + vaesenclast xmm0, xmm0, XMMWORD PTR [rsi + rdx * 2] + CHECK_XMM xmm0, last_mem_twice + + vmovdqu xmm1, XMMWORD PTR [rip + state0] + vaesenc xmm1, xmm1, xmm2 + CHECK_XMM xmm1, enc0 + + vmovdqu xmm1, XMMWORD PTR [rip + state0] + vmovdqu xmm2, XMMWORD PTR [rip + key] + vaesenc xmm2, xmm1, xmm2 + CHECK_XMM xmm2, enc0 + + vmovdqu xmm0, XMMWORD PTR [rip + state0] + vmovdqu xmm4, XMMWORD PTR [rip + ones] + vinsertf128 ymm0, ymm0, xmm4, 1 + vaesenc xmm0, xmm1, xmm2 + vextractf128 xmm3, ymm0, 1 + vptest xmm3, xmm3 + je .Lupper_zero + inc DWORD PTR [rip + fail_count] +.Lupper_zero: + + vmovdqu xmm0, XMMWORD PTR [rip + state0] + vmovdqu xmm2, XMMWORD PTR [rip + key] + .rept 9 + vaesenc xmm0, xmm0, xmm2 + .endr + vaesenclast xmm0, xmm0, xmm2 + CHECK_XMM xmm0, chain + + vmovdqu xmm0, XMMWORD PTR [rip + state0] + vmovdqu xmm1, XMMWORD PTR [rip + state1] + vaesenc xmm0, xmm0, xmm2 + vaesenc xmm1, xmm1, xmm2 + CHECK_XMM xmm0, enc0 + CHECK_XMM xmm1, enc1 + + vmovdqu xmm0, XMMWORD PTR [rip + state0] + vmovdqu xmm1, XMMWORD PTR [rip + state1] + vaesenclast xmm0, xmm0, xmm2 + vaesenclast xmm1, xmm1, xmm2 + CHECK_XMM xmm0, last0 + CHECK_XMM xmm1, last1 + + vmovdqu xmm0, XMMWORD PTR [rip + state0] + mov ecx, 500000 +.Laot_loop: + vaesenc xmm0, xmm0, xmm2 + dec ecx + jne .Laot_loop + + vzeroupper + mov edi, DWORD PTR [rip + fail_count] + mov eax, 60 + syscall + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/vaes-ymm-rounds.S b/tests/integration/vaes-ymm-rounds.S new file mode 100644 index 00000000000..fb23158dd91 --- /dev/null +++ b/tests/integration/vaes-ymm-rounds.S @@ -0,0 +1,124 @@ +.intel_syntax noprefix +.global _start + +.macro CHECK_YMM reg, expected + vmovdqu YMMWORD PTR [rip + actual], \reg + lea rsi, [rip + actual] + lea rdi, [rip + \expected] + mov ecx, 4 + cld + repe cmpsq + je .Lcheck_done\@ + inc DWORD PTR [rip + fail_count] +.Lcheck_done\@: +.endm + +.section .data +fail_count: .long 0 + +.section .bss +.balign 32 +actual: .zero 32 + +.section .rodata +.balign 32 +state: .byte 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 + .byte 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +key: .byte 32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47 + .byte 48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63 +expected_enc: + .byte 0x4a,0x4b,0x7e,0x66,0x08,0x48,0x15,0x76,0x98,0xf0,0x77,0x4a,0x0b,0xb1,0x0f,0x73 + .byte 0x36,0x74,0xa5,0x16,0xb3,0x97,0x7b,0x7e,0xb7,0x77,0xb1,0x41,0x6a,0xd8,0x44,0x00 +expected_last: + .byte 0x43,0x4a,0x45,0x55,0xd6,0x24,0x8d,0x5c,0x18,0xfe,0x5d,0xee,0xd2,0x51,0x41,0x04 + .byte 0xfa,0x68,0x90,0xf3,0xce,0xe1,0x44,0x4a,0x95,0x9d,0xf3,0xcb,0xa0,0xbf,0x79,0x90 +expected_enc_twice: + .byte 0x1b,0xd8,0xdd,0xe0,0x8e,0xbf,0x23,0xeb,0x2c,0xd2,0x11,0xaa,0x3b,0x46,0x65,0x0f + .byte 0x12,0x1f,0x91,0xba,0xb6,0xc3,0x51,0xe0,0x27,0xab,0xf0,0x41,0x37,0xe0,0x72,0x97 +expected_last_twice: + .byte 0x3a,0x17,0x6e,0xd1,0xd2,0x9e,0xa5,0xdb,0x85,0xf8,0x44,0x61,0x99,0xfb,0x73,0x07 + .byte 0x1d,0xc9,0x3f,0x53,0xbf,0x6b,0x80,0x3a,0x12,0x31,0x5a,0xed,0xdc,0x78,0x25,0x20 + +.section .text +_start: + mov r10d, 500000 +.Lx8_loop: + vmovdqu ymm0, YMMWORD PTR [rip + state] + vmovdqu ymm1, YMMWORD PTR [rip + state] + vmovdqu ymm2, YMMWORD PTR [rip + state] + vmovdqu ymm3, YMMWORD PTR [rip + state] + vmovdqu ymm4, YMMWORD PTR [rip + state] + vmovdqu ymm5, YMMWORD PTR [rip + state] + vmovdqu ymm6, YMMWORD PTR [rip + state] + vmovdqu ymm7, YMMWORD PTR [rip + state] + vmovdqu ymm8, YMMWORD PTR [rip + key] + vmovdqu ymm15, YMMWORD PTR [rip + key] + vaesenc ymm0, ymm0, ymm8 + vaesenc ymm1, ymm1, ymm8 + vaesenc ymm2, ymm2, ymm8 + vaesenc ymm3, ymm3, ymm8 + vaesenc ymm4, ymm4, ymm8 + vaesenc ymm5, ymm5, ymm8 + vaesenc ymm6, ymm6, ymm8 + vaesenc ymm7, ymm7, ymm8 + dec r10d + jne .Lx8_loop + CHECK_YMM ymm0, expected_enc + CHECK_YMM ymm1, expected_enc + CHECK_YMM ymm2, expected_enc + CHECK_YMM ymm3, expected_enc + CHECK_YMM ymm4, expected_enc + CHECK_YMM ymm5, expected_enc + CHECK_YMM ymm6, expected_enc + CHECK_YMM ymm7, expected_enc + CHECK_YMM ymm15, key + + vmovdqu ymm1, YMMWORD PTR [rip + state] + vmovdqu ymm2, YMMWORD PTR [rip + key] + vaesenc ymm0, ymm1, ymm2 + CHECK_YMM ymm0, expected_enc + + vmovdqu ymm1, YMMWORD PTR [rip + state] + vaesenc ymm1, ymm1, ymm2 + CHECK_YMM ymm1, expected_enc + + vmovdqu ymm1, YMMWORD PTR [rip + state] + vaesenc ymm0, ymm1, YMMWORD PTR [rip + key] + CHECK_YMM ymm0, expected_enc + + vmovdqu ymm1, YMMWORD PTR [rip + state] + vaesenclast ymm0, ymm1, ymm2 + CHECK_YMM ymm0, expected_last + + vmovdqu ymm1, YMMWORD PTR [rip + state] + vaesenclast ymm1, ymm1, ymm2 + CHECK_YMM ymm1, expected_last + + vmovdqu ymm1, YMMWORD PTR [rip + state] + vaesenclast ymm0, ymm1, YMMWORD PTR [rip + key] + CHECK_YMM ymm0, expected_last + + lea rsi, [rip + key] + xor edx, edx + vmovdqu ymm0, YMMWORD PTR [rip + state] + vaesenc ymm0, ymm0, YMMWORD PTR [rsi + rdx * 2] + vaesenc ymm0, ymm0, YMMWORD PTR [rsi + rdx * 2] + CHECK_YMM ymm0, expected_enc_twice + + lea rsi, [rip + key] + vmovdqu ymm0, YMMWORD PTR [rip + state] + vaesenclast ymm0, ymm0, YMMWORD PTR [rsi + rdx * 2] + vaesenclast ymm0, ymm0, YMMWORD PTR [rsi + rdx * 2] + CHECK_YMM ymm0, expected_last_twice + + vmovdqu ymm1, YMMWORD PTR [rip + state] + vmovdqu ymm2, YMMWORD PTR [rip + key] + vaesenc ymm2, ymm1, ymm2 + CHECK_YMM ymm2, expected_enc + + vzeroupper + mov edi, DWORD PTR [rip + fail_count] + mov eax, 60 + syscall + +.section .note.GNU-stack,"",@progbits