From 70ac49588cd06335eb0a6e2e1444921c527fad99 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:26:06 +0800 Subject: [PATCH 01/10] latx: synchronize MXCSR exception state with FCSR Install the complete load/store exception mapping, replace hardware flags on load, honor guest exception masks without the x87 diagnostic override, and clear stale Cause. Preserve MMX state around the status helper. No software Denormal-input detection is introduced. Includes emitter-model and x86 guest regression tests. The 524,288-case host model passes; LoongArch execution is pending. Reorganized from 8235cda3ac, 6aa7692c1c and 73d84958d8. --- target/i386/latx/translator/tr-fctrl.c | 61 +++++++++- tests/latx/mxcsr-exceptions.c | 60 ++++++++++ tests/latx/test-mxcsr-fcsr.py | 151 +++++++++++++++++++++++++ 3 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 tests/latx/mxcsr-exceptions.c create mode 100644 tests/latx/test-mxcsr-fcsr.py diff --git a/target/i386/latx/translator/tr-fctrl.c b/target/i386/latx/translator/tr-fctrl.c index 36773c5f273..b10acca544f 100644 --- a/target/i386/latx/translator/tr-fctrl.c +++ b/target/i386/latx/translator/tr-fctrl.c @@ -129,6 +129,43 @@ void update_fcsr_by_cw(IR2_OPND cw) ra_free_temp(fcsr); } +static void update_fcsr_exceptions_by_mxcsr(IR2_OPND mxcsr) +{ + IR2_OPND fcsr = ra_alloc_itemp(); + IR2_OPND mapped = ra_alloc_itemp(); + IR2_OPND invalid = ra_alloc_itemp(); + + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + + /* MXCSR flags [5:2,0] (P,U,O,Z,I) map to FCSR [20:16] + * (V,Z,O,U,I). Replace, rather than accumulate, on LDMXCSR so that + * a subsequent STMXCSR cannot resurrect cleared hardware flags. + * MXCSR Denormal (bit 1) has no FCSR counterpart. */ + la_bitrev_w(mapped, mxcsr); + la_bstrpick_w(mapped, mapped, 29, 26); + la_bstrins_w(fcsr, mapped, FCSR_OFF_FLAGS_Z, FCSR_OFF_FLAGS_I); + la_bstrins_w(fcsr, mxcsr, FCSR_OFF_FLAGS_V, FCSR_OFF_FLAGS_V); + + /* Masks [12:9,7] use the opposite polarity to FCSR Enables. + * Do not use update_fcsr_enable(): its x87 diagnostic override can + * force traps even when the guest MXCSR masks those exceptions. */ + la_srli_w(mapped, mxcsr, 7); + la_bstrpick_w(invalid, mapped, 0, 0); + la_bitrev_w(mapped, mapped); + la_bstrpick_w(mapped, mapped, 29, 26); + la_bstrins_w(mapped, invalid, FCSR_OFF_EN_V, FCSR_OFF_EN_V); + la_xori(mapped, mapped, FCSR_ENABLE_SET); + la_bstrins_w(fcsr, mapped, FCSR_OFF_EN_V, FCSR_OFF_EN_I); + + /* LDMXCSR starts a new exception state, not a new FP operation. */ + la_bstrins_w(fcsr, zero_ir2_opnd, FCSR_OFF_CAUSE_V, FCSR_OFF_CAUSE_I); + la_movgr2fcsr(fcsr_ir2_opnd, fcsr); + + ra_free_temp(invalid); + ra_free_temp(mapped); + ra_free_temp(fcsr); +} + void update_sw_by_fcsr(IR2_OPND sw_opnd) { int status_offset = lsenv_offset_of_status_word(lsenv); @@ -166,6 +203,26 @@ void update_sw_by_fcsr(IR2_OPND sw_opnd) ra_free_temp(temp1); } +static void update_mxcsr_flags_by_fcsr(IR2_OPND mxcsr) +{ + IR2_OPND fcsr = ra_alloc_itemp(); + IR2_OPND flags = ra_alloc_itemp(); + IR2_OPND mapped = ra_alloc_itemp(); + + la_movfcsr2gr(fcsr, fcsr_ir2_opnd); + la_bitrev_w(flags, fcsr); + la_srli_d(flags, flags, 11); + la_bstrpick_d(mapped, flags, 0, 0); + la_or(mxcsr, mxcsr, mapped); + la_bstrpick_d(mapped, flags, 4, 1); + la_slli_d(mapped, mapped, 2); + la_or(mxcsr, mxcsr, mapped); + + ra_free_temp(mapped); + ra_free_temp(flags); + ra_free_temp(fcsr); +} + bool translate_fnstcw(IR1_INST *pir1) { /* 1. load the value of fpu control word */ @@ -213,6 +270,7 @@ bool translate_stmxcsr(IR1_INST *pir1) lsassert(offset <= 0x7ff); la_ld_wu(mxcsr_opnd, env_ir2_opnd, offset); + update_mxcsr_flags_by_fcsr(mxcsr_opnd); /* 2. store the value of the mxcsr register state to the dest_opnd */ store_ireg_to_ir1(mxcsr_opnd, ir1_get_opnd(pir1, 0), false); @@ -231,8 +289,9 @@ bool translate_ldmxcsr(IR1_INST *pir1) /* 2. store the value into the env->mxcsr */ lsassert(offset <= 0x7ff); la_st_w(new_mxcsr, env_ir2_opnd, offset); + update_fcsr_exceptions_by_mxcsr(new_mxcsr); - tr_gen_call_to_helper1((ADDR)update_mxcsr_status, 1, + tr_gen_call_to_helper1((ADDR)update_mxcsr_status, 0, LOAD_HELPER_UPDATE_MXCSR_STATUS); return true; diff --git a/tests/latx/mxcsr-exceptions.c b/tests/latx/mxcsr-exceptions.c new file mode 100644 index 00000000000..670d221d2ea --- /dev/null +++ b/tests/latx/mxcsr-exceptions.c @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: GPL-2.0-only + * + * Build on x86-64: cc -O2 -mavx tests/latx/mxcsr-exceptions.c -o mxcsr-exceptions + * Run natively, then under LATX on LoongArch with LATX_ENABLE_FCSR_EXC=0/1. + * Requires AVX. A SIGFPE with all exceptions masked is itself a test failure. + */ +#include +#include + +#define CHECK_SEQUENCE(LOAD, STORE, DIV, ONE, ZERO) do { \ + const uint32_t clean = 0x1f80, one = 0x3f800000, zero = 0; \ + uint32_t saved, raised, cleared; \ + __asm__ volatile( \ + "stmxcsr %[saved]\n\t" \ + LOAD " %[clean]\n\t" \ + "movss %[one], %%xmm0\n\t" \ + "movss %[zero], %%xmm1\n\t" \ + DIV " " ZERO ", " ONE "\n\t" \ + STORE " %[raised]\n\t" \ + LOAD " %[clean]\n\t" \ + STORE " %[cleared]\n\t" \ + "ldmxcsr %[saved]\n\t" \ + : [saved] "=m" (saved), [raised] "=m" (raised), \ + [cleared] "=m" (cleared) \ + : [clean] "m" (clean), [one] "m" (one), [zero] "m" (zero) \ + : "xmm0", "xmm1", "memory"); \ + if (raised != (clean | 4) || cleared != clean) { \ + fprintf(stderr, LOAD ": raised=%08x cleared=%08x\n", \ + raised, cleared); \ + ++failures; \ + } \ + /* Exercise replacement of every exception-flag combination. */ \ + for (uint32_t flags = 0; flags < 64; ++flags) { \ + uint32_t input = clean | flags, output; \ + __asm__ volatile( \ + "stmxcsr %[saved]\n\t" \ + LOAD " %[input]\n\t" \ + STORE " %[output]\n\t" \ + "ldmxcsr %[saved]\n\t" \ + : [saved] "=m" (saved), [output] "=m" (output) \ + : [input] "m" (input) : "memory"); \ + if (output != input) { \ + fprintf(stderr, LOAD ": input=%08x output=%08x\n", \ + input, output); \ + ++failures; \ + } \ + } \ +} while (0) + +int main(void) +{ + int failures = 0; + CHECK_SEQUENCE("ldmxcsr", "stmxcsr", "divss", "%%xmm0", "%%xmm1"); + CHECK_SEQUENCE("vldmxcsr", "vstmxcsr", "vdivss", + "%%xmm0, %%xmm0", "%%xmm1"); + printf("MXCSR exception tests: %s (%d failures)\n", + failures ? "FAIL" : "PASS", failures); + return failures != 0; +} diff --git a/tests/latx/test-mxcsr-fcsr.py b/tests/latx/test-mxcsr-fcsr.py new file mode 100644 index 00000000000..b2b25225b77 --- /dev/null +++ b/tests/latx/test-mxcsr-fcsr.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Host-independent semantics test of the actual MXCSR IR2 emitter. + +Run: python3 tests/latx/test-mxcsr-fcsr.py +Use --revision HEAD^ to check the pre-fix emitter without changing checkout. +Requires a C compiler. This models emitted integer instructions, not LASX +execution, helper-call preservation, or floating-point exception delivery. +""" + +import argparse +import os +from pathlib import Path +import re +import shlex +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = "target/i386/latx/translator/tr-fctrl.c" + + +def function(source, name): + match = re.search(r"static void " + name + r"\([^)]*\)\s*\{", source) + if not match: + raise RuntimeError(f"missing function: {name}") + depth = 1 + end = match.end() + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + + +HARNESS = r""" +#include +#include +#include +typedef int IR2_OPND; +static uint64_t regs[32]; +static uint32_t hardware_fcsr; +static int next_reg, option_enable_fcsr_exc; +enum { zero_ir2_opnd = 0, fcsr_ir2_opnd = 31 }; +static int ra_alloc_itemp(void) { return next_reg++; } +static void ra_free_temp(int r) { (void)r; } +static uint64_t sext32(uint32_t v) { return (int64_t)(int32_t)v; } +static void la_movfcsr2gr(int d, int s) +{ (void)s; regs[d] = sext32(hardware_fcsr); } +static void la_movgr2fcsr(int d, int s) +{ (void)d; hardware_fcsr = regs[s]; } +static void la_bitrev_w(int d, int s) +{ + uint32_t x = regs[s], r = 0; + for (int i = 0; i < 32; ++i) { r = (r << 1) | (x & 1); x >>= 1; } + regs[d] = sext32(r); +} +static void la_bstrpick_w(int d, int s, int hi, int lo) +{ regs[d] = sext32(((uint32_t)regs[s] >> lo) & ((1ULL << (hi-lo+1))-1)); } +static void la_bstrpick_d(int d, int s, int hi, int lo) +{ regs[d] = (regs[s] >> lo) & ((1ULL << (hi-lo+1))-1); } +static void la_bstrins_w(int d, int s, int hi, int lo) +{ + uint32_t mask = ((1ULL << (hi-lo+1))-1) << lo; + regs[d] = sext32(((uint32_t)regs[d] & ~mask) | + (((uint32_t)regs[s] << lo) & mask)); +} +static void la_srli_w(int d, int s, int n) +{ regs[d] = sext32((uint32_t)regs[s] >> n); } +static void la_srli_d(int d, int s, int n) { regs[d] = regs[s] >> n; } +static void la_slli_d(int d, int s, int n) { regs[d] = regs[s] << n; } +static void la_xori(int d, int s, int imm) { regs[d] = regs[s] ^ imm; } +static void la_andi(int d, int s, int imm) { regs[d] = regs[s] & imm; } +static void la_or(int d, int a, int b) { regs[d] = regs[a] | regs[b]; } +""" + +CHECK = r""" +int main(void) +{ + /* Explicit architecture mapping, independent of emitter bit reversal. */ + const int x86_bits[] = {5, 4, 3, 2, 0}; + const uint32_t initial[] = {0, 0x1f1f031f, 0x001f0000, 0x150a020a}; + const uint32_t changed = (31u << 24) | (31u << 16) | 31u; + unsigned failures[2] = {0, 0}; + unsigned cases = 0; + for (int option = 0; option < 2; ++option) { + option_enable_fcsr_exc = option; + for (uint32_t mxcsr = 0; mxcsr < 65536; ++mxcsr) { + for (unsigned k = 0; k < sizeof(initial)/sizeof(initial[0]); ++k) { + uint32_t expected = initial[k] & ~changed; + for (int i = 0; i < 5; ++i) { + expected |= ((mxcsr >> x86_bits[i]) & 1) << (16+i); + expected |= (((mxcsr >> (x86_bits[i]+7)) & 1) ^ 1) << i; + } + memset(regs, 0, sizeof(regs)); + next_reg = 2; + regs[1] = mxcsr; + hardware_fcsr = initial[k]; + LOAD_MXCSR(1); + if (hardware_fcsr != expected || regs[1] != mxcsr) { + if (failures[option]++ == 0) { + fprintf(stderr, "option=%d mxcsr=%04x old=%08x " + "actual=%08x expected=%08x\n", option, + mxcsr, initial[k], hardware_fcsr, expected); + } + } + /* The next STMXCSR must not bring back cleared flags. */ + next_reg = 2; + update_mxcsr_flags_by_fcsr(1); + if (regs[1] != mxcsr) ++failures[option]; + ++cases; + } + } + } + printf("%u cases; failures with option=0: %u, option=1: %u\n", + cases, failures[0], failures[1]); + return !!(failures[0] || failures[1]); +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--revision") + args = parser.parse_args() + if args.revision: + source = subprocess.check_output( + ["git", "show", f"{args.revision}:{SOURCE}"], cwd=ROOT, text=True) + else: + source = (ROOT / SOURCE).read_text() + # Follow the helper actually called by translate_ldmxcsr(). + load_body = source[source.index("bool translate_ldmxcsr("):] + name = re.search(r"(update_fcsr_\w+)\(new_mxcsr\);", load_body).group(1) + definitions = (ROOT / "target/i386/latx/include/env.h").read_text() + constants = "\n".join(re.findall( + r"^#define (?:FCSR_|X87_CR_OFF_)\w+\s+(?:0x[0-9a-fA-F]+|\d+)\s*$", + definitions, re.MULTILINE)) + emitter = function(source, "update_fcsr_enable") + "\n" + function(source, name) + emitter += "\n" + function(source, "update_mxcsr_flags_by_fcsr") + with tempfile.TemporaryDirectory(prefix="latx-mxcsr-test-") as directory: + path = Path(directory) + (path / "test.c").write_text(constants + "\n" + HARNESS + emitter + + f"\n#define LOAD_MXCSR {name}\n" + CHECK) + subprocess.run(shlex.split(os.environ.get("CC", "cc")) + + ["-std=c11", "-O2", "-Wall", "-Wextra", + "-Wno-unused-function", str(path / "test.c"), + "-o", str(path / "test")], check=True) + return subprocess.run([str(path / "test")]).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) From e3f31e84bb7d8caa395c6aa52548f7ae4f6e009e Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:26:22 +0800 Subject: [PATCH 02/10] latx: preserve F16C result and exception state Keep VCVTPS2PH register upper lanes clear and propagate helper exception status into MXCSR without clobbering MMX. Register the helper's AOT relocation together with its use. Use the final implementation without the redundant per-lane clear. Reorganized from d63662659c, 79d1b38107 and d4012d8f19; no behavior change relative to ef9dde911e. --- target/i386/latx/include/aot.h | 1 + target/i386/latx/sbt/aot.c | 1 + target/i386/latx/translator/tr-simd-f16c.c | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index f72613a814d..a5c9f3a62cf 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -177,6 +177,7 @@ typedef enum aot_rel_kind { LOAD_HELPER_TRACE_SESSION_BEGIN, LOAD_HELPER_UPDATE_MXCSR_STATUS, + LOAD_HELPER_UPDATE_MXCSR, LOAD_HELPER_FPATAN, LOAD_HELPER_FPTAN, LOAD_HELPER_FPREM, diff --git a/target/i386/latx/sbt/aot.c b/target/i386/latx/sbt/aot.c index 3aa6292cfac..dfada7f4d85 100644 --- a/target/i386/latx/sbt/aot.c +++ b/target/i386/latx/sbt/aot.c @@ -1209,6 +1209,7 @@ static void* relkind_to_fixup_addr[] = { [LOAD_VPAES_DEC_TABLES_XV] = (void *)latx_vpaes_dec_tables_xv, [LOAD_HELPER_TRACE_SESSION_BEGIN] = trace_session_begin, [LOAD_HELPER_UPDATE_MXCSR_STATUS] = update_mxcsr_status, + [LOAD_HELPER_UPDATE_MXCSR] = helper_update_mxcsr, [LOAD_HELPER_FPATAN] = helper_fpatan, [LOAD_HELPER_FPTAN] = helper_fptan, [LOAD_HELPER_FPREM] = helper_fprem, diff --git a/target/i386/latx/translator/tr-simd-f16c.c b/target/i386/latx/translator/tr-simd-f16c.c index f29ecf95a3a..a293f87a003 100644 --- a/target/i386/latx/translator/tr-simd-f16c.c +++ b/target/i386/latx/translator/tr-simd-f16c.c @@ -75,7 +75,10 @@ bool translate_vcvtps2ph(IR1_INST *pir1) if (!ir1_opnd_is_mem(opnd0)) { int d = ir1_opnd_base_reg_num(opnd0); + IR2_OPND dest = ra_alloc_xmm(d); + tr_gen_call_to_helper_pcmpxstrx((ADDR)helper_func, d, s, imm, rel_kind); + set_high128_xreg_to_zero(dest); } else { int d = (s + 1) & 7; IR2_OPND temp = ra_alloc_ftemp(); @@ -92,6 +95,8 @@ bool translate_vcvtps2ph(IR1_INST *pir1) la_xvor_v(dest, temp, temp); } + tr_gen_call_to_helper1((ADDR)helper_update_mxcsr, 0, + LOAD_HELPER_UPDATE_MXCSR); return true; } #endif From f831f23fd4f3a6f4cea6045c344e9303a5feb7f4 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:26:36 +0800 Subject: [PATCH 03/10] latx: decode VEX.W AVX string compare forms Decode the equivalent W=0 bytes for Capstone and restore the original W bit and instruction bytes in IR1. Reorganized from 01b000d63d. Also retain the final scalar-compare assertion indentation; that line has no semantic change. All content matches the corresponding paths in ef9dde911e. --- target/i386/latx/ir1/ir1.c | 20 ++++++++++++++++++++ target/i386/latx/translator/tr-avx-cmp.c | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/target/i386/latx/ir1/ir1.c b/target/i386/latx/ir1/ir1.c index 41ba0ee23b2..ed22bad1873 100644 --- a/target/i386/latx/ir1/ir1.c +++ b/target/i386/latx/ir1/ir1.c @@ -336,6 +336,9 @@ ADDRX ir1_disasm(IR1_INST *ir1, uint8_t *addr, ADDRX t_pc, int ir1_num, void *pi struct la_dt_insn *info; uint32_t nop = 0x401f0f; uint64_t nop_5 = 0x441f0f; + uint8_t vex_w0[15]; + const uint8_t *original_addr = addr; + bool restore_vex_w = false; if (((*((uint32_t *)addr)) & 0xf8ffffff) == 0xc81e0ff3) { //repleace endbr32/rdsspd with 4 bytes nop, just a temporary solution addr = (uint8_t *)&nop; @@ -351,6 +354,18 @@ ADDRX ir1_disasm(IR1_INST *ir1, uint8_t *addr, ADDRX t_pc, int ir1_num, void *pi addr = (uint8_t *)&nop_5; } #endif + /* + * Capstone rejects VEX.W=1 for the four AVX string compare opcodes, + * although W selects their 64-bit-length form. Decode the equivalent + * W=0 encoding, then retain W and the original bytes in IR1. + */ + if (addr[0] == 0xc4 && (addr[1] & 0x1f) == 0x03 && + (addr[2] & 0x85) == 0x81 && addr[3] >= 0x60 && addr[3] <= 0x63) { + memcpy(vex_w0, addr, sizeof(vex_w0)); + vex_w0[2] &= ~0x80; + addr = vex_w0; + restore_vex_w = true; + } /* FIXME:the count parameter in cs_disasm is 1, it means we translte 1 insn at a time, * there should be a performance improvement if we increase the number, but * for now there are some problems if we change it. It will be settled later. @@ -370,6 +385,11 @@ ADDRX ir1_disasm(IR1_INST *ir1, uint8_t *addr, ADDRX t_pc, int ir1_num, void *pi exit(-1); } + if (restore_vex_w) { + info->x86.rex |= 0x08; + memcpy(info->bytes, original_addr, info->size); + } + disassemble_trace_cmp(addr, 15, (uint64_t)t_pc, 1, info, CODEIS64); ir1->_eflag = 0; diff --git a/target/i386/latx/translator/tr-avx-cmp.c b/target/i386/latx/translator/tr-avx-cmp.c index 5461b36dbdd..f13b54d4132 100644 --- a/target/i386/latx/translator/tr-avx-cmp.c +++ b/target/i386/latx/translator/tr-avx-cmp.c @@ -3878,7 +3878,7 @@ bool translate_vcmpss(IR1_INST *pir1) lsassert(ir1_opnd_num(pir1) == 4 && ir1_opnd_is_imm(ir1_get_opnd(pir1, 3))); lsassert((ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0)) && - ir1_opnd_is_xmm(ir1_get_opnd(pir1, 1)))); + ir1_opnd_is_xmm(ir1_get_opnd(pir1, 1)))); uint8 predicate = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0x1f; switch (predicate) { case 0: From b5cb9fe059f3ec304be2033774013b31829612df Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:26:36 +0800 Subject: [PATCH 04/10] latx: preserve the full YMM value for zero-offset VPALIGNR Use the full-width copy in the zero-offset path. Reorganized from 9c3aaaa9c3 without changing the final implementation. --- target/i386/latx/translator/tr-avx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 7c809154e59..1de2cb58233 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -4546,7 +4546,7 @@ bool translate_vpalignr(IR1_INST * pir1) { } else { /* slow path */ if (imm == 0) { - la_vori_b(dest, src2, 0); + la_xvori_b(dest, src2, 0); } else { IR2_OPND temp_src2 = ra_alloc_ftemp(); la_xvbsrl_v(temp_src2, src2, imm); From f6d609eaeb650e8ed2f8952fcafef1d9d13ff8f3 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:26:49 +0800 Subject: [PATCH 05/10] latx: accept maximum valid AVX right-shift counts Retain the boundary correction from 8e22df54e8 as an independent integer-instruction fix. --- target/i386/latx/translator/tr-avx-shift.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/i386/latx/translator/tr-avx-shift.c b/target/i386/latx/translator/tr-avx-shift.c index 6e75e3ece61..bf399d8b9ec 100644 --- a/target/i386/latx/translator/tr-avx-shift.c +++ b/target/i386/latx/translator/tr-avx-shift.c @@ -194,7 +194,7 @@ bool translate_vpsrlx(IR1_INST * pir1) { IR2_OPND count = ra_alloc_itemp(); IR2_OPND max = ra_alloc_itemp(); - la_addi_d(max, zero_ir2_opnd, max_count); + la_addi_d(max, zero_ir2_opnd, max_count + 1); la_vpickve2gr_d(count, src2, 0); la_blt(count, max, label_shift); la_xvxor_v(dest, dest, dest); From 1c98b0b601af797d2d4c75ad83a2d61ab5f074dd Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:27:14 +0800 Subject: [PATCH 06/10] latx: repair arithmetic NaNs with non-signaling vector classification Introduce the final scalar/packed classification helper and use it for add, subtract, multiply, divide, square root and reciprocal estimates. Preserve original sources for NaN payload/sign selection and confine scalar repair to the active lane. Include the 12,600-case host emitter model. The optimized implementation is introduced directly: no per-lane predecessor, FCMP-based repair or software Denormal-input detection is added. Arithmetic/helper portions reorganized from d0b2f2a0c5, 4aad1fab5d, f034607870, 94ab1c9239 and ef9dde911e. FMA and reductions follow separately. Runtime validation remains pending. --- target/i386/latx/translator/tr-avx.c | 288 +++++++++++++++++++++--- tests/latx/test-nan-repair.py | 321 +++++++++++++++++++++++++++ 2 files changed, 580 insertions(+), 29 deletions(-) create mode 100644 tests/latx/test-nan-repair.py diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 1de2cb58233..4d09e3dc708 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -13,6 +13,12 @@ #include "pclmul.h" #ifdef CONFIG_LATX_AVX_OPT +static void lasx_fp_fix_unary_nan(IR2_OPND result, IR2_OPND src, + bool double_precision, int lanes); +static void lasx_fp_fix_binary_nan(IR2_OPND result, IR2_OPND src1, + IR2_OPND src2, bool double_precision, + int lanes); + bool translate_vaddpd(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vaddpd_lsx(pir1); @@ -28,15 +34,17 @@ bool translate_vaddpd(IR1_INST * pir1) { IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src1 = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); IR2_OPND src2; + IR2_OPND temp = ra_alloc_ftemp(); if (ir1_opnd_is_ymm(opnd0)) { src2 = load_freg256_from_ir1(opnd2); - la_xvfadd_d(dest, src1, src2); + la_xvfadd_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 4); + la_xvori_b(dest, temp, 0); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); la_vfadd_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 2); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -58,15 +66,17 @@ bool translate_vaddps(IR1_INST * pir1) { IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src1 = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); IR2_OPND src2; + IR2_OPND temp = ra_alloc_ftemp(); if (ir1_opnd_is_ymm(opnd0)) { src2 = load_freg256_from_ir1(opnd2); - la_xvfadd_s(dest, src1, src2); + la_xvfadd_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 8); + la_xvori_b(dest, temp, 0); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); la_vfadd_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 4); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -86,6 +96,7 @@ bool translate_vaddsd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_fadd_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 1); la_vshuf4i_d(temp, src1, 0xc); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); @@ -105,6 +116,7 @@ bool translate_vaddss(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_fadd_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 1); if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { la_xvori_b(dest, src1, 0); @@ -129,15 +141,17 @@ bool translate_vsubpd(IR1_INST * pir1) { IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src1 = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); IR2_OPND src2; + IR2_OPND temp = ra_alloc_ftemp(); if (ir1_opnd_is_ymm(opnd0)) { src2 = load_freg256_from_ir1(opnd2); - la_xvfsub_d(dest, src1, src2); + la_xvfsub_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 4); + la_xvori_b(dest, temp, 0); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); la_vfsub_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 2); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -159,15 +173,17 @@ bool translate_vsubps(IR1_INST * pir1) { IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src1 = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); IR2_OPND src2; + IR2_OPND temp = ra_alloc_ftemp(); if (ir1_opnd_is_ymm(opnd0)) { src2 = load_freg256_from_ir1(opnd2); - la_xvfsub_s(dest, src1, src2); + la_xvfsub_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 8); + la_xvori_b(dest, temp, 0); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); la_vfsub_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 4); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -188,6 +204,7 @@ bool translate_vsubsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); la_fsub_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 1); la_vshuf4i_d(temp, src1, 0xc); la_xvori_b(dest, temp, 0); set_high128_xreg_to_zero(dest); @@ -207,6 +224,7 @@ bool translate_vsubss(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_fsub_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 1); if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { la_xvori_b(dest, src1, 0); @@ -231,15 +249,17 @@ bool translate_vmulpd(IR1_INST * pir1) { IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src1 = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); IR2_OPND src2; + IR2_OPND temp = ra_alloc_ftemp(); if (ir1_opnd_is_ymm(opnd0)) { src2 = load_freg256_from_ir1(opnd2); - la_xvfmul_d(dest, src1, src2); + la_xvfmul_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 4); + la_xvori_b(dest, temp, 0); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); la_vfmul_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 2); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -261,15 +281,17 @@ bool translate_vmulps(IR1_INST * pir1) { IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src1 = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); IR2_OPND src2; + IR2_OPND temp = ra_alloc_ftemp(); if (ir1_opnd_is_ymm(opnd0)) { src2 = load_freg256_from_ir1(opnd2); - la_xvfmul_s(dest, src1, src2); + la_xvfmul_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 8); + la_xvori_b(dest, temp, 0); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); la_vfmul_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 4); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -288,6 +310,7 @@ bool translate_vmulsd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_fmul_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 1); la_vshuf4i_d(temp, src1, 0xc); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); @@ -307,6 +330,7 @@ bool translate_vmulss(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_fmul_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 1); if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { la_xvori_b(dest, src1, 0); @@ -331,21 +355,171 @@ bool translate_vdivpd(IR1_INST * pir1) { IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src1 = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); IR2_OPND src2; + IR2_OPND temp = ra_alloc_ftemp(); if (ir1_opnd_is_ymm(opnd0)) { src2 = load_freg256_from_ir1(opnd2); - la_xvfdiv_d(dest, src1, src2); + la_xvfdiv_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 4); + la_xvori_b(dest, temp, 0); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); la_vfdiv_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 2); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } return true; } +/* + * Repair scalar and packed results without changing FCSR. FCLASS returns + * 1 for SNaN, 2 for QNaN, and a single bit >= 4 for every other class; + * an unsigned <= 3 comparison therefore produces an all-one NaN mask. + * FCMP must not be used here: it changes Cause and can signal on SNaN. + * The result mask provides the all-non-NaN fast exit. In ordinary + * arithmetic an input NaN necessarily has a NaN result, so its source mask + * also confines the selection. Reductions with masked output lanes pass + * mask_sources_with_result to retain that confinement explicitly. A caller + * may instead preserve the original result and restore its finite lanes after + * the selection; this costs two instructions regardless of source count. + * Sources must still hold the original operands and must not alias result. + */ +static void lasx_fp_fix_vector_nan_from_sources(IR2_OPND result, + const IR2_OPND *sources, + int source_count, + bool double_precision, + int lanes, + const int *source_shuffles, + bool destructive_shuffles, + bool mask_sources_with_result, + IR2_OPND preserve_non_nan) +{ + bool lasx = double_precision ? lanes == 4 : lanes == 8; + IR2_INST *(*classify)(IR2_OPND, IR2_OPND); + IR2_INST *(*nan_mask)(IR2_OPND, IR2_OPND, int); + IR2_INST *(*seteqz)(IR2_OPND, IR2_OPND); + IR2_INST *(*shift_left)(IR2_OPND, IR2_OPND, int); + IR2_INST *(*set_bit)(IR2_OPND, IR2_OPND, int); + IR2_INST *(*and)(IR2_OPND, IR2_OPND, IR2_OPND); + IR2_INST *(*select)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); + IR2_INST *(*shuffle)(IR2_OPND, IR2_OPND, int) = NULL; + IR2_OPND result_nan = ra_alloc_ftemp(); + IR2_OPND work = ra_alloc_ftemp(); + IR2_OPND done = ra_alloc_label(); + + if (lasx) { + classify = double_precision ? la_xvfclass_d : la_xvfclass_s; + nan_mask = double_precision ? la_xvslei_du : la_xvslei_wu; + seteqz = la_xvseteqz_v; + shift_left = double_precision ? la_xvslli_d : la_xvslli_w; + set_bit = double_precision ? la_xvbitseti_d : la_xvbitseti_w; + and = la_xvand_v; + select = la_xvbitsel_v; + if (source_shuffles != NULL) { + shuffle = double_precision ? la_xvshuf4i_d : la_xvshuf4i_w; + } + } else { + classify = double_precision ? la_vfclass_d : la_vfclass_s; + nan_mask = double_precision ? la_vslei_du : la_vslei_wu; + seteqz = la_vseteqz_v; + shift_left = double_precision ? la_vslli_d : la_vslli_w; + set_bit = double_precision ? la_vbitseti_d : la_vbitseti_w; + and = la_vand_v; + select = la_vbitsel_v; + if (source_shuffles != NULL) { + shuffle = double_precision ? la_vshuf4i_d : la_vshuf4i_w; + } + } + + classify(result_nan, result); + nan_mask(result_nan, result_nan, 3); + if (lanes == 1) { + /* Inactive scalar lanes must not trigger repair or be modified. + * FCLASS is non-signaling, so classifying their bits is harmless. */ + if (double_precision) { + la_xvpickve_d(result_nan, result_nan, 0); + } else { + la_xvpickve_w(result_nan, result_nan, 0); + } + } + seteqz(fcc0_ir2_opnd, result_nan); + la_bcnez(fcc0_ir2_opnd, done); + + if (!ir2_opnd_is_none(&preserve_non_nan)) { + la_xvori_b(preserve_non_nan, result, 0); + } + + /* all-one NaN mask << 22/51 is x86's indefinite qNaN. */ + shift_left(work, result_nan, double_precision ? 51 : 22); + select(result, result, work, result_nan); + + /* Later selections are overwritten by earlier x86 source operands. */ + IR2_OPND shuffled = ir2_opnd_new_none(); + + if (source_shuffles != NULL && !destructive_shuffles) { + shuffled = ra_alloc_ftemp(); + } + for (int source = source_count - 1; source >= 0; --source) { + IR2_OPND source_opnd = sources[source]; + + if (source_shuffles != NULL && source_shuffles[source] >= 0) { + if (destructive_shuffles) { + shuffle(source_opnd, source_opnd, source_shuffles[source]); + } else { + shuffle(shuffled, source_opnd, source_shuffles[source]); + source_opnd = shuffled; + } + } + classify(work, source_opnd); + nan_mask(work, work, 3); + if (lanes == 1 || + (mask_sources_with_result && ir2_opnd_is_none(&preserve_non_nan))) { + and(work, work, result_nan); + } + select(result, result, source_opnd, work); + } + + /* Set quiet only in lanes selected by the original result NaN mask. */ + set_bit(work, result, double_precision ? 51 : 22); + select(result, result, work, result_nan); + if (!ir2_opnd_is_none(&preserve_non_nan)) { + select(result, preserve_non_nan, result, result_nan); + } + + la_label(done); + ra_free_temp_auto(shuffled); + ra_free_temp(work); + ra_free_temp(result_nan); +} + +static void lasx_fp_fix_nan_from_sources(IR2_OPND result, + const IR2_OPND *sources, + int source_count, + bool double_precision, int lanes) +{ + lasx_fp_fix_vector_nan_from_sources(result, sources, source_count, + double_precision, lanes, NULL, + false, false, ir2_opnd_new_none()); +} + +static void lasx_fp_fix_unary_nan(IR2_OPND result, IR2_OPND src, + bool double_precision, int lanes) +{ + IR2_OPND sources[] = { src }; + + lasx_fp_fix_nan_from_sources(result, sources, 1, double_precision, lanes); +} + +static void lasx_fp_fix_binary_nan(IR2_OPND result, IR2_OPND src1, + IR2_OPND src2, bool double_precision, + int lanes) +{ + IR2_OPND sources[] = { src1, src2 }; + + lasx_fp_fix_nan_from_sources(result, sources, 2, double_precision, lanes); +} + bool translate_vdivps(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vdivps_lsx(pir1); @@ -361,18 +535,40 @@ bool translate_vdivps(IR1_INST * pir1) { IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src1 = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); IR2_OPND src2; + IR2_OPND src1_for_nan = src1; + IR2_OPND src2_for_nan; + bool saved_src1 = false; + bool saved_src2 = false; if (ir1_opnd_is_ymm(opnd0)) { src2 = load_freg256_from_ir1(opnd2); + src2_for_nan = src2; + if (ir2_opnd_cmp(&dest, &src1)) { + src1_for_nan = ra_alloc_ftemp(); + la_xvori_b(src1_for_nan, src1, 0); + saved_src1 = true; + } + if (ir2_opnd_cmp(&dest, &src2)) { + src2_for_nan = ra_alloc_ftemp(); + la_xvori_b(src2_for_nan, src2, 0); + saved_src2 = true; + } la_xvfdiv_s(dest, src1, src2); + lasx_fp_fix_binary_nan(dest, src1_for_nan, src2_for_nan, false, 8); } else if (ir1_opnd_is_xmm(opnd0)) { IR2_OPND temp = ra_alloc_ftemp(); src2 = load_freg128_from_ir1(opnd2); la_vfdiv_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 4); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } + if (saved_src2) + ra_free_temp(src2_for_nan); + if (saved_src1) + ra_free_temp(src1_for_nan); + ra_free_temp_auto(src2); return true; } @@ -387,6 +583,7 @@ bool translate_vdivsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_fdiv_d(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, true, 1); la_vshuf4i_d(temp, src1, 0xc); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); @@ -407,6 +604,7 @@ bool translate_vdivss(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); la_fdiv_s(temp, src1, src2); + lasx_fp_fix_binary_nan(temp, src1, src2, false, 1); if (ir1_opnd_base_reg_num(opnd0) != ir1_opnd_base_reg_num(opnd1)) { la_xvori_b(dest, src1, 0); } @@ -427,13 +625,23 @@ bool translate_vsqrtpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - la_xvfsqrt_d(dest, src); + if (ir2_opnd_cmp(&dest, &src)) { + IR2_OPND saved_src = ra_alloc_ftemp(); + la_xvori_b(saved_src, src, 0); + la_xvfsqrt_d(dest, saved_src); + lasx_fp_fix_unary_nan(dest, saved_src, true, 4); + ra_free_temp(saved_src); + } else { + la_xvfsqrt_d(dest, src); + lasx_fp_fix_unary_nan(dest, src, true, 4); + } } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND temp = ra_alloc_ftemp(); la_vfsqrt_d(temp, src); + lasx_fp_fix_unary_nan(temp, src, true, 2); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -452,13 +660,23 @@ bool translate_vsqrtps(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - la_xvfsqrt_s(dest, src); + if (ir2_opnd_cmp(&dest, &src)) { + IR2_OPND saved_src = ra_alloc_ftemp(); + la_xvori_b(saved_src, src, 0); + la_xvfsqrt_s(dest, saved_src); + lasx_fp_fix_unary_nan(dest, saved_src, false, 8); + ra_free_temp(saved_src); + } else { + la_xvfsqrt_s(dest, src); + lasx_fp_fix_unary_nan(dest, src, false, 8); + } } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND temp = ra_alloc_ftemp(); la_vfsqrt_s(temp, src); + lasx_fp_fix_unary_nan(temp, src, false, 4); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -479,6 +697,7 @@ bool translate_vsqrtsd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); la_fsqrt_d(temp, src2); + lasx_fp_fix_unary_nan(temp, src2, true, 1); if (ir1_opnd_base_reg_num(opnd0) != ir1_opnd_base_reg_num(opnd1)) { la_xvori_b(dest, src1, 0); } @@ -501,6 +720,7 @@ bool translate_vsqrtss(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp = ra_alloc_ftemp(); la_fsqrt_s(temp, src2); + lasx_fp_fix_unary_nan(temp, src2, false, 1); if (ir1_opnd_base_reg_num(opnd0) != ir1_opnd_base_reg_num(opnd1)) { la_xvori_b(dest, src1, 0); } @@ -3559,6 +3779,7 @@ bool translate_vrsqrtss(IR1_INST * pir1) { /* this x86 instruction has no exception ,so we need mask all exception */ la_frsqrt_s(temp, src2); + lasx_fp_fix_unary_nan(temp, src2, false, 1); la_xvori_b(dest, src1, 0x0); la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); @@ -3575,14 +3796,18 @@ bool translate_vrsqrtps(IR1_INST * pir1) { IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); + IR2_OPND temp = ra_alloc_ftemp(); + int lanes = ir1_opnd_is_xmm(opnd0) ? 4 : 8; /* this x86 instruction has no exception ,so we need mask all exception */ if (ir1_opnd_is_xmm(opnd0)) { - la_vfrsqrt_s(dest, src); - set_high128_xreg_to_zero(dest); + la_vfrsqrt_s(temp, src); + set_high128_xreg_to_zero(temp); } else { - la_xvfrsqrt_s(dest, src); + la_xvfrsqrt_s(temp, src); } + lasx_fp_fix_unary_nan(temp, src, false, lanes); + la_xvori_b(dest, temp, 0); return true; } @@ -3596,14 +3821,18 @@ bool translate_vrcpps(IR1_INST * pir1) { IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); + IR2_OPND temp = ra_alloc_ftemp(); + int lanes = ir1_opnd_is_xmm(opnd0) ? 4 : 8; /* this x86 instruction has no exception ,so we need mask all exception */ if (ir1_opnd_is_xmm(opnd0)) { - la_vfrecip_s(dest, src); - set_high128_xreg_to_zero(dest); + la_vfrecip_s(temp, src); + set_high128_xreg_to_zero(temp); } else { - la_xvfrecip_s(dest, src); + la_xvfrecip_s(temp, src); } + lasx_fp_fix_unary_nan(temp, src, false, lanes); + la_xvori_b(dest, temp, 0); return true; } @@ -3620,6 +3849,7 @@ bool translate_vrcpss(IR1_INST * pir1) { /* this x86 instruction has no exception ,so we need mask all exception */ la_frecip_s(temp, src2); + lasx_fp_fix_unary_nan(temp, src2, false, 1); la_xvori_b(dest, src1, 0x0); la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); diff --git a/tests/latx/test-nan-repair.py b/tests/latx/test-nan-repair.py new file mode 100644 index 00000000000..2685c9a4de0 --- /dev/null +++ b/tests/latx/test-nan-repair.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +"""Compile the actual NaN IR2 emitter and interpret its emitted instructions. + +Run with Python 3 and CC (default cc); --revision HEAD tests the old emitter. +This is a bit-exact host model, not a replacement for LoongArch runtime tests. +Vector mask, shuffle, pick and select semantics follow the LoongArch ISA. +""" +import argparse +import json +import os +from pathlib import Path +import random +import re +import shlex +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = "target/i386/latx/translator/tr-avx.c" +MASK64 = (1 << 64) - 1 + + +def extract(source, name): + m = re.search(r"(?:static )?void " + name + r"\([^;]*?\)\s*\{", source) + if not m: + return "" + end, depth = m.end(), 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[m.start():end] + + +def programs(source): + names = ["lasx_fp_fix_nan_from_sources_lane", + "lasx_fp_fix_packed_nan_from_sources", + "lasx_fp_fix_vector_nan_from_sources", + "lasx_fp_fix_nan_from_sources"] + body = "\n".join(extract(source, name) for name in names) + vector = ("lasx_fp_fix_vector_nan_from_sources" if extract(source, names[2]) + else "lasx_fp_fix_packed_nan_from_sources") + header = r''' +#include +#include +#include +#include +#include +typedef int IR2_OPND; +typedef int IR2_INST; +static int nf, ni, nl, live, peak; +enum { zero_ir2_opnd = 99, fcc0_ir2_opnd = 90 }; +static int ra_alloc_ftemp(void) { ++live; if (live > peak) peak = live; return nf++; } +static int ra_alloc_itemp(void) { return ni++; } +static int ra_alloc_label(void) { return nl++; } +static void ra_free_temp(int r) { if (r >= 16 && r < 32) --live; } +static void ra_free_temp_auto(int r) { if (r >= 0) ra_free_temp(r); } +static int ir2_opnd_new_none(void) { return -1; } +static bool ir2_opnd_is_none(const int *r) { return *r == -1; } +static void li_d(int r, uint64_t v) { printf("li_d %d %llu\n", r, (unsigned long long)v); } +''' + arity = {"label": 1, "b": 1, "bnez": 2, "bcnez": 2, + "beq": 3, "bne": 3, "and": 3, "or": 3, "movfr2gr_d": 2} + for prefix in ("v", "xv"): + arity.update({prefix + name: n for name, n in { + "fcmp_cond_s": 4, "fcmp_cond_d": 4, + "mskltz_w": 2, "mskltz_d": 2, + "fclass_s": 2, "fclass_d": 2, + "slei_wu": 3, "slei_du": 3, "seteqz_v": 2, + "slli_w": 3, "slli_d": 3, "bitseti_w": 3, "bitseti_d": 3, + "and_v": 3, "ori_b": 3, "bitsel_v": 4, + "shuf4i_w": 3, "shuf4i_d": 3, + }.items()}) + arity.update({"xv" + n: 3 for n in ("pickve_w", "pickve_d", + "pickve2gr_w", "pickve2gr_d", "pickve2gr_du", + "insgr2vr_w", "insgr2vr_d")}) + templates = json.loads((ROOT / "target/i386/latx/inst_template.json").read_text()) + signatures = {name.replace(".", "_"): len(spec["opnd"]) + for name, spec in templates.items() if "opnd" in spec} + for name in sorted(set(re.findall(r"\bla_(\w+)\b", body))): + n = arity[name] + if name.startswith(("v", "xv")): + assert signatures[name] == n, (name, signatures.get(name), n) + args = ", ".join(f"int a{i}" for i in range(n)) + fmt = " %d" * n + values = ", ".join(f"a{i}" for i in range(n)) + header += (f'static IR2_INST *la_{name}({args}) ' + f'{{ printf("{name}{fmt}\\n", {values}); return NULL; }}\n') + main = r''' +int main(void) { + for (int dp = 0; dp < 2; ++dp) { + for (int form = 0; form < 3; ++form) { + int lanes = form == 0 ? 1 : (dp ? 2 : 4) * form; + for (int count = 1; count <= 3; ++count) { + for (int aliases = 0; aliases < 2; ++aliases) { + int sources[] = {1, aliases ? 1 : 2, aliases ? 1 : 3}; + nf = 16; ni = 32; nl = 1000; live = peak = 0; + printf("CASE %d %d %d %d 0\n", dp, lanes, count, aliases); + lasx_fp_fix_nan_from_sources(0, sources, count, dp, lanes); + printf("END %d\n", peak); + } + } + } + for (int wide = 0; wide < (dp ? 1 : 2); ++wide) { + for (int masked = 0; masked < 2; ++masked) { + int lanes = (dp ? 2 : 4) * (wide + 1); + int sources[] = {1, 1, 1, 1}; + const int pd[] = {1, 1}; + const int ps[] = {0xb1, -1, 0x1b, 0x4e}; + nf = 16; ni = 32; nl = 1000; live = peak = 0; + printf("CASE %d %d %d 1 %d\n", dp, lanes, dp ? 2 : 4, masked + 1); + VECTOR(0, sources, dp ? 2 : 4, dp, lanes, dp ? pd : ps, + dp, masked, !dp && masked ? 4 : -1); + printf("END %d\n", peak); + } + } + } +} +'''.replace("VECTOR", vector) + with tempfile.TemporaryDirectory(prefix="latx-nan-model-") as directory: + path = Path(directory) + (path / "emit.c").write_text(header + body + main) + subprocess.run(shlex.split(os.environ.get("CC", "cc")) + + ["-std=c11", "-O2", "-Wall", "-Wextra", + "-Wno-unused-function", str(path / "emit.c"), + "-o", str(path / "emit")], check=True) + output = subprocess.check_output([str(path / "emit")], text=True) + result = [] + for line in output.splitlines(): + op, *args = line.split() + args = list(map(int, args)) + if op == "CASE": + case, code = args, [] + elif op == "END": + result.append((case, code, args[0])) + else: + code.append((op, args)) + return result + + +def nan(x, bits): + frac = 52 if bits == 64 else 23 + return (x & ((1 << (bits - 1)) - 1)) > (((1 << (bits-frac-1))-1) << frac) + + +def classify(x, bits): + frac = 52 if bits == 64 else 23 + sign = x >> (bits-1) + if nan(x, bits): + return 2 if x & (1 << (frac-1)) else 1 + exp = (x >> frac) & ((1 << (bits-frac-1))-1) + if exp == (1 << (bits-frac-1))-1: + return 1 << (2 if sign else 6) + if exp: + return 1 << (3 if sign else 7) + return 1 << ((4 if sign else 8) if x & ((1 << frac)-1) else (5 if sign else 9)) + + +def lane(v, i, bits): + return (v >> (bits*i)) & ((1 << bits)-1) + + +def pack(values, bits): + return sum(v << (i*bits) for i, v in enumerate(values)) + + +def execute(code, initial, fcsr): + r = dict(initial) + r[99] = 0 + labels = {a[0]: i for i, (op, a) in enumerate(code) if op == "label"} + pc, steps = 0, 0 + while pc < len(code): + op, a = code[pc] + pc += 1 + if op == "label": + continue + steps += 1 + d = a[0] + get = lambda reg: r.get(reg, 0) + if op == "b": + pc = labels[d] + elif op in ("bnez", "bcnez"): + if get(d): pc = labels[a[1]] + elif op in ("beq", "bne"): + if (get(d) == get(a[1])) == (op == "beq"): pc = labels[a[2]] + elif op == "li_d": r[d] = a[1] + elif op == "and": r[d] = get(a[1]) & get(a[2]) + elif op == "or": r[d] = get(a[1]) | get(a[2]) + elif op == "movfr2gr_d": r[d] = get(a[1]) & MASK64 + elif "pickve2gr" in op: + bits = 32 if op.endswith("_w") else 64 + v = lane(get(a[1]), a[2], bits) + r[d] = v | (MASK64 ^ ((1 << bits)-1)) if bits == 32 and v >> 31 else v + elif "insgr2vr" in op: + bits = 32 if op.endswith("_w") else 64 + mask = ((1 << bits)-1) << (a[2]*bits) + r[d] = (get(d) & ~mask) | ((get(a[1]) << (a[2]*bits)) & mask) + elif op in ("xvpickve_w", "xvpickve_d"): + r[d] = lane(get(a[1]), a[2], 32 if op.endswith("_w") else 64) + else: + width = 256 if op.startswith("xv") else 128 + insn = op[2:] if width == 256 else op[1:] + bits = 64 if insn.endswith(("_d", "_du")) else 32 + full, mask = (1 << width)-1, (1 << bits)-1 + x = get(a[1]) + if insn == "seteqz_v": + r[d] = int((x & full) == 0) + continue + if insn == "bitsel_v": v = (x & ~get(a[3])) | (get(a[2]) & get(a[3])) + elif insn == "and_v": v = x & get(a[2]) + elif insn == "ori_b": + assert a[2] == 0 + v = x + elif insn.startswith("mskltz"): + v = sum((lane(x, i, bits) >> (bits-1)) << + ((i*bits//128)*128 + i % (128//bits)) for i in range(width//bits)) + elif insn.startswith("shuf4i"): + values = [] + for i in range(width//bits): + base = i // (128//bits) * (128//bits) + idx = (a[2] >> (2*(i % (128//bits)))) & 3 + if bits == 64: + values.append(lane(get(d) if idx < 2 else x, base + idx % 2, bits)) + else: values.append(lane(x, base+idx, bits)) + v = pack(values, bits) + else: + values = [] + if insn.startswith("fcmp"): + fcsr &= ~(31 << 24) + for i in range(width//bits): + item = lane(x, i, bits) + if insn.startswith("fclass"): value = classify(item, bits) + elif insn.startswith("fcmp"): + assert a[3] == 8 and a[1] == a[2] + value = mask if nan(item, bits) else 0 + if classify(item, bits) == 1: fcsr |= (1 << 20) | (1 << 28) + elif insn.startswith("slei"): value = mask if item <= a[2] else 0 + elif insn.startswith("slli"): value = (item << a[2]) & mask + elif insn.startswith("bitseti"): value = item | (1 << a[2]) + else: raise ValueError(op) + values.append(value) + v = pack(values, bits) + # Model LSX's low 128 bits; the caller clears VEX upper halves. + r[d] = (get(d) & ~full) | (v & full) + return r, fcsr, steps + + +def check(all_programs): + rng = random.Random(438) + failures, cases, counts = {}, 0, [] + for (dp, lanes, count, aliases, mode), code, peak in all_programs: + bits = 64 if dp else 32 + width = max(128, lanes*bits) + quiet = 1 << (51 if dp else 22) + infinity = 0x7ff0000000000000 if dp else 0x7f800000 + indefinite = (1 << (bits-1)) | infinity | quiet + samples = [0, 1, infinity-1, infinity, infinity|1, + infinity|quiet|0x123, infinity|quiet|0x456, + (1 << (bits-1)) | infinity | 0x789] + for trial in range(300): + sources = {reg: [rng.choice(samples) if trial < 240 else rng.getrandbits(bits) + for _ in range(width//bits)] for reg in (1, 2, 3)} + original = [rng.choice(samples) for _ in range(width//bits)] + if not mode and trial < width//bits: + # Precisely one NaN, including only in the upper LASX half + # or in an inactive scalar lane. No other lane can hide it. + sources = {reg: [0] * (width//bits) for reg in (1, 2, 3)} + sources[1][trial] = infinity | 1 + original = [0] * (width//bits) + original[trial] = indefinite + # Ordinary arithmetic propagates any input NaN to its result. + for i in range(lanes): + if mode: + enabled = trial & (1 << (i % (128//bits))) + original[i] = 0 if mode == 2 and not enabled else indefinite + elif any(nan(sources[1 if aliases else j+1][i], bits) for j in range(count)): + original[i] = indefinite + expected = original[:] + for i in range(lanes): + if not nan(original[i], bits): continue + candidates = [] + for j in range(count): + if mode: + if dp: idx = i if j == 0 else i ^ 1 + else: + shuffle = [0xb1, 0xe4, 0x1b, 0x4e][j] + idx = (i//4)*4 + ((shuffle >> ((i%4)*2)) & 3) + candidates.append(sources[1][idx]) + else: candidates.append(sources[1 if aliases else j+1][i]) + expected[i] = next((x | quiet for x in candidates if nan(x, bits)), indefinite) + initial = {0: pack(original, bits), **{k: pack(v, bits) for k, v in sources.items()}} + # Include upper-lane-only NaNs, inactive scalar SNaNs, and nonzero Cause. + fcsr = [0, 0x08080000, 0x1010011f][trial % 3] + out, status, steps = execute(code, initial, fcsr) + checks = {"value": (out[0] & ((1 << width)-1)) == pack(expected, bits), + "fcsr": status == fcsr, + "sources": all(out[k] == initial[k] for k in (1, 2, 3))} + for key, ok in checks.items(): + if not ok: + failures[key] = failures.get(key, 0)+1 + cases += 1 + if not aliases and not mode: + finite = {0: 0, 1: 0, 2: 0, 3: 0} + _, _, fast = execute(code, finite, 0) + counts.append({"bits": bits, "lanes": lanes, "sources": count, + "emitter_ops": sum(op != "label" for op, _ in code), + "finite_ops": fast, "helper_ftemps_peak": peak}) + print(json.dumps({"cases": cases, "failures": failures, "counts": counts}, indent=2)) + return bool(failures) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--revision") + args = parser.parse_args() + source = (subprocess.check_output(["git", "show", f"{args.revision}:{SOURCE}"], + cwd=ROOT, text=True) if args.revision else (ROOT / SOURCE).read_text()) + return check(programs(source)) + + +if __name__ == "__main__": + raise SystemExit(main()) From f1101c63dd19634979173bcbf5990f1a1ce0dc4a Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:27:38 +0800 Subject: [PATCH 07/10] latx: preserve original FMA operands for NaN repair Connect scalar, packed and mixed add/sub FMA paths to the final shared NaN repair. Keep original inputs available across sign changes and destination/source aliases, including third-source aliases. Export the helper together with its cross-file callers and remove superseded in-line negative-FMA NaN handling. Reorganized from d0b2f2a0c5, b5fb77269e and 9e8638a269; final behavior matches ef9dde911e. --- target/i386/latx/include/translate.h | 5 + target/i386/latx/translator/tr-avx.c | 213 ++++++++-------------- target/i386/latx/translator/tr-simd-fma.c | 88 +++++++-- 3 files changed, 149 insertions(+), 157 deletions(-) diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 38200bc4298..bfd8f68c3ed 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -34,6 +34,11 @@ bool TRANS_FUNC(name)(IR1_INST * pir1) #define TRANS_FUNC_GEN(opcode, function) \ TRANS_FUNC_GEN_REAL(opcode, TRANS_FUNC(function)) +#ifdef CONFIG_LATX_AVX_OPT +void lasx_fp_fix_fma_nan(IR2_OPND result, IR2_OPND src1, IR2_OPND src2, + IR2_OPND src3, bool double_precision, int lanes); +#endif + /* * AVX integer operations that can be evaluated independently in each 128-bit * half. The operation token is also used by the LSX implementation generator. diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 4d09e3dc708..1569a8b29be 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -520,6 +520,14 @@ static void lasx_fp_fix_binary_nan(IR2_OPND result, IR2_OPND src1, lasx_fp_fix_nan_from_sources(result, sources, 2, double_precision, lanes); } +void lasx_fp_fix_fma_nan(IR2_OPND result, IR2_OPND src1, IR2_OPND src2, + IR2_OPND src3, bool double_precision, int lanes) +{ + IR2_OPND sources[] = { src1, src2, src3 }; + + lasx_fp_fix_nan_from_sources(result, sources, 3, double_precision, lanes); +} + bool translate_vdivps(IR1_INST * pir1) { if (!option_enable_lasx) { return translate_vdivps_lsx(pir1); @@ -2795,6 +2803,7 @@ bool translate_vfmaddxxxss(IR1_INST * pir1) { } la_fmadd_s(temp, temp1, temp2, temp3); + lasx_fp_fix_fma_nan(temp, temp1, temp2, temp3, false, 1); la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); return true; @@ -2829,6 +2838,7 @@ bool translate_vfmaddxxxsd(IR1_INST * pir1) { } la_fmadd_d(temp, temp1, temp2, temp3); + lasx_fp_fix_fma_nan(temp, temp1, temp2, temp3, true, 1); la_xvinsve0_d(dest, temp, 0); set_high128_xreg_to_zero(dest); return true; @@ -2868,6 +2878,8 @@ bool translate_vfmaddxxxpd(IR1_INST * pir1) { tr_inst = ir1_opnd_is_xmm(opnd0) ? la_vfmadd_d : la_xvfmadd_d; tr_inst(temp, temp1, temp2, temp3); + lasx_fp_fix_fma_nan(temp, temp1, temp2, temp3, true, + ir1_opnd_is_xmm(opnd0) ? 2 : 4); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -2908,6 +2920,8 @@ bool translate_vfmaddxxxps(IR1_INST * pir1) { tr_inst = ir1_opnd_is_xmm(opnd0) ? la_vfmadd_s : la_xvfmadd_s; tr_inst(temp, temp1, temp2, temp3); + lasx_fp_fix_fma_nan(temp, temp1, temp2, temp3, false, + ir1_opnd_is_xmm(opnd0) ? 4 : 8); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -2943,6 +2957,7 @@ bool translate_vfmsubxxxss(IR1_INST * pir1) { } la_fmsub_s(temp, temp1, temp2, temp3); + lasx_fp_fix_fma_nan(temp, temp1, temp2, temp3, false, 1); la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); return true; @@ -2977,6 +2992,7 @@ bool translate_vfmsubxxxsd(IR1_INST * pir1) { } la_fmsub_d(temp, temp1, temp2, temp3); + lasx_fp_fix_fma_nan(temp, temp1, temp2, temp3, true, 1); la_xvinsve0_d(dest, temp, 0); set_high128_xreg_to_zero(dest); return true; @@ -3016,6 +3032,8 @@ bool translate_vfmsubxxxpd(IR1_INST * pir1) { tr_inst = ir1_opnd_is_xmm(opnd0) ? la_vfmsub_d : la_xvfmsub_d; tr_inst(temp, temp1, temp2, temp3); + lasx_fp_fix_fma_nan(temp, temp1, temp2, temp3, true, + ir1_opnd_is_xmm(opnd0) ? 2 : 4); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3056,6 +3074,8 @@ bool translate_vfmsubxxxps(IR1_INST * pir1) { tr_inst = ir1_opnd_is_xmm(opnd0) ? la_vfmsub_s : la_xvfmsub_s; tr_inst(temp, temp1, temp2, temp3); + lasx_fp_fix_fma_nan(temp, temp1, temp2, temp3, false, + ir1_opnd_is_xmm(opnd0) ? 4 : 8); la_xvori_b(dest, temp, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); @@ -3073,6 +3093,7 @@ bool translate_vfnmaddxxxss(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -3091,20 +3112,9 @@ bool translate_vfnmaddxxxss(IR1_INST * pir1) { } la_fneg_s(temp, temp1); - la_fmadd_s(temp, temp, temp2, temp3); - - IR2_OPND label_over = ra_alloc_label(); - /* check if result is NaN */ - la_fcmp_cond_s(fcc0_ir2_opnd, temp, temp, 0x8); - - /* if no NaN happend, compution done */ - la_bceqz(fcc0_ir2_opnd, label_over); - - /* if INVALID NaN did happen, use original operands to generate correct NaN */ - la_fmsub_s(temp, src1, src2, dest); - - la_label(label_over); - la_xvinsve0_w(dest, temp, 0); + la_fmadd_s(result, temp, temp2, temp3); + lasx_fp_fix_fma_nan(result, temp1, temp2, temp3, false, 1); + la_xvinsve0_w(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -3120,6 +3130,7 @@ bool translate_vfnmaddxxxsd(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -3138,20 +3149,9 @@ bool translate_vfnmaddxxxsd(IR1_INST * pir1) { } la_fneg_d(temp, temp1); - la_fmadd_d(temp, temp, temp2, temp3); - - IR2_OPND label_over = ra_alloc_label(); - /* check if result is NaN */ - la_fcmp_cond_d(fcc0_ir2_opnd, temp, temp, 0x8); - - /* if no NaN happend, compution done */ - la_bceqz(fcc0_ir2_opnd, label_over); - - /* if INVALID NaN did happen, use original operands to generate correct NaN */ - la_fmsub_d(temp, src1, src2, dest); - - la_label(label_over); - la_xvinsve0_d(dest, temp, 0); + la_fmadd_d(result, temp, temp2, temp3); + lasx_fp_fix_fma_nan(result, temp1, temp2, temp3, true, 1); + la_xvinsve0_d(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -3171,7 +3171,9 @@ bool translate_vfnmaddxxxpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; + IR2_OPND ftemp = ra_alloc_ftemp(); IR2_OPND itemp = ra_alloc_itemp(); IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -3193,34 +3195,20 @@ bool translate_vfnmaddxxxpd(IR1_INST * pir1) { /* change the first operand sign bit*/ la_lu52i_d(itemp, zero_ir2_opnd, 0x800); - la_vinsgr2vr_d(temp, itemp, 0); - la_xvreplve0_d(temp, temp); - la_xvxor_v(temp, temp1, temp); - /* compute the result*/ - tr_inst(temp, temp, temp2, temp3); - - IR2_OPND mask = ra_alloc_ftemp(); - IR2_OPND src1_temp = ra_alloc_ftemp(); - IR2_OPND src2_temp = ra_alloc_ftemp(); - IR2_OPND src3_temp = ra_alloc_ftemp(); - - /* check if result is NaN */ - la_xvfcmp_cond_d(mask, temp, temp, 0x8); - la_xvand_v(src1_temp, mask, dest); - la_xvand_v(src2_temp, mask, src1); - la_xvand_v(src3_temp, mask, src2); - tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); - - la_xvbitsel_v(temp, temp, src1_temp, mask); - la_xvori_b(dest, temp, 0); + la_vinsgr2vr_d(ftemp, itemp, 0); + la_xvreplve0_d(ftemp, ftemp); + la_xvxor_v(temp, temp1, ftemp); + tr_inst(result, temp, temp2, temp3); + lasx_fp_fix_fma_nan(result, temp1, temp2, temp3, true, + ir1_opnd_is_xmm(opnd0) ? 2 : 4); + la_xvori_b(dest, result, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - ra_free_temp(src1_temp); - ra_free_temp(src2_temp); - ra_free_temp(src3_temp); + ra_free_temp(result); ra_free_temp(temp); - ra_free_temp(mask); + ra_free_temp(ftemp); + ra_free_temp(itemp); return true; } @@ -3239,6 +3227,7 @@ bool translate_vfnmaddxxxps(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR2_OPND ftemp = ra_alloc_ftemp(); IR2_OPND itemp = ra_alloc_itemp(); @@ -3264,30 +3253,16 @@ bool translate_vfnmaddxxxps(IR1_INST * pir1) { la_xvreplve0_w(ftemp, ftemp); la_xvxor_v(temp, temp1, ftemp); - /* compute the result*/ - tr_inst(temp, temp, temp2, temp3); - - IR2_OPND mask = ra_alloc_ftemp(); - IR2_OPND src1_temp = ra_alloc_ftemp(); - IR2_OPND src2_temp = ra_alloc_ftemp(); - IR2_OPND src3_temp = ra_alloc_ftemp(); - - /* check if result is NaN */ - la_xvfcmp_cond_s(mask, temp, temp, 0x8); - la_xvand_v(src1_temp, mask, dest); - la_xvand_v(src2_temp, mask, src1); - la_xvand_v(src3_temp, mask, src2); - tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); - - la_xvbitsel_v(temp, temp, src1_temp, mask); - la_xvori_b(dest, temp, 0); + tr_inst(result, temp, temp2, temp3); + lasx_fp_fix_fma_nan(result, temp1, temp2, temp3, false, + ir1_opnd_is_xmm(opnd0) ? 4 : 8); + la_xvori_b(dest, result, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - ra_free_temp(src1_temp); - ra_free_temp(src2_temp); - ra_free_temp(src3_temp); + ra_free_temp(result); ra_free_temp(temp); - ra_free_temp(mask); + ra_free_temp(ftemp); + ra_free_temp(itemp); return true; } @@ -3302,6 +3277,7 @@ bool translate_vfnmsubxxxss(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -3320,20 +3296,9 @@ bool translate_vfnmsubxxxss(IR1_INST * pir1) { } la_fneg_s(temp, temp1); - la_fmsub_s(temp, temp, temp2, temp3); - - IR2_OPND label_over = ra_alloc_label(); - /* check if result is NaN */ - la_fcmp_cond_s(fcc0_ir2_opnd, temp, temp, 0x8); - - /* if no NaN happend, compution done */ - la_bceqz(fcc0_ir2_opnd, label_over); - - /* if INVALID NaN did happen, use original operands to generate correct NaN */ - la_fmsub_s(temp, src1, src2, dest); - - la_label(label_over); - la_xvinsve0_w(dest, temp, 0); + la_fmsub_s(result, temp, temp2, temp3); + lasx_fp_fix_fma_nan(result, temp1, temp2, temp3, false, 1); + la_xvinsve0_w(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -3349,6 +3314,7 @@ bool translate_vfnmsubxxxsd(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -3368,20 +3334,9 @@ bool translate_vfnmsubxxxsd(IR1_INST * pir1) { /* if the dest is NaN , translation may take mistake * becasue x86 vfnmsub and 3a5000 vfmsub may produce different NaN*/ la_fneg_d(temp, temp1); - la_fmsub_d(temp, temp, temp2, temp3); - - IR2_OPND label_over = ra_alloc_label(); - /* check if result is NaN */ - la_fcmp_cond_d(fcc0_ir2_opnd, temp, temp, 0x8); - - /* if no NaN happend, compution done */ - la_bceqz(fcc0_ir2_opnd, label_over); - - /* if INVALID NaN did happen, use original operands to generate correct NaN */ - la_fmsub_d(temp, src1, src2, dest); - - la_label(label_over); - la_xvinsve0_d(dest, temp, 0); + la_fmsub_d(result, temp, temp2, temp3); + lasx_fp_fix_fma_nan(result, temp1, temp2, temp3, true, 1); + la_xvinsve0_d(dest, result, 0); set_high128_xreg_to_zero(dest); return true; } @@ -3401,6 +3356,7 @@ bool translate_vfnmsubxxxpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR2_OPND ftemp = ra_alloc_ftemp(); IR2_OPND itemp = ra_alloc_itemp(); @@ -3426,29 +3382,16 @@ bool translate_vfnmsubxxxpd(IR1_INST * pir1) { la_vinsgr2vr_d(ftemp, itemp, 0); la_xvreplve0_d(ftemp, ftemp); la_xvxor_v(temp, temp1, ftemp); - tr_inst(temp, temp, temp2, temp3); - - IR2_OPND mask = ra_alloc_ftemp(); - IR2_OPND src1_temp = ra_alloc_ftemp(); - IR2_OPND src2_temp = ra_alloc_ftemp(); - IR2_OPND src3_temp = ra_alloc_ftemp(); - - /* check if result is NaN */ - la_xvfcmp_cond_d(mask, temp, temp, 0x8); - la_xvand_v(src1_temp, mask, dest); - la_xvand_v(src2_temp, mask, src1); - la_xvand_v(src3_temp, mask, src2); - tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); - - la_xvbitsel_v(temp, temp, src1_temp, mask); - la_xvori_b(dest, temp, 0); + tr_inst(result, temp, temp2, temp3); + lasx_fp_fix_fma_nan(result, temp1, temp2, temp3, true, + ir1_opnd_is_xmm(opnd0) ? 2 : 4); + la_xvori_b(dest, result, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - ra_free_temp(src1_temp); - ra_free_temp(src2_temp); - ra_free_temp(src3_temp); + ra_free_temp(result); ra_free_temp(temp); - ra_free_temp(mask); + ra_free_temp(ftemp); + ra_free_temp(itemp); return true; } @@ -3467,6 +3410,7 @@ bool translate_vfnmsubxxxps(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR2_OPND ftemp = ra_alloc_ftemp(); IR2_OPND itemp = ra_alloc_itemp(); @@ -3491,29 +3435,16 @@ bool translate_vfnmsubxxxps(IR1_INST * pir1) { la_vinsgr2vr_w(ftemp, itemp, 0); la_xvreplve0_w(ftemp, ftemp); la_xvxor_v(temp, temp1, ftemp); - tr_inst(temp, temp, temp2, temp3); - - IR2_OPND mask = ra_alloc_ftemp(); - IR2_OPND src1_temp = ra_alloc_ftemp(); - IR2_OPND src2_temp = ra_alloc_ftemp(); - IR2_OPND src3_temp = ra_alloc_ftemp(); - - /* check if result is NaN */ - la_xvfcmp_cond_s(mask, temp, temp, 0x8); - la_xvand_v(src1_temp, mask, dest); - la_xvand_v(src2_temp, mask, src1); - la_xvand_v(src3_temp, mask, src2); - tr_inst(src1_temp, src2_temp, src3_temp, src1_temp); - - la_xvbitsel_v(temp, temp, src1_temp, mask); - la_xvori_b(dest, temp, 0); + tr_inst(result, temp, temp2, temp3); + lasx_fp_fix_fma_nan(result, temp1, temp2, temp3, false, + ir1_opnd_is_xmm(opnd0) ? 4 : 8); + la_xvori_b(dest, result, 0); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); - ra_free_temp(src1_temp); - ra_free_temp(src2_temp); - ra_free_temp(src3_temp); + ra_free_temp(result); ra_free_temp(temp); - ra_free_temp(mask); + ra_free_temp(ftemp); + ra_free_temp(itemp); return true; } diff --git a/target/i386/latx/translator/tr-simd-fma.c b/target/i386/latx/translator/tr-simd-fma.c index b392d206311..dcca7db51d0 100644 --- a/target/i386/latx/translator/tr-simd-fma.c +++ b/target/i386/latx/translator/tr-simd-fma.c @@ -17,26 +17,37 @@ bool translate_vfmaddsubxxxps(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); + IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); lsassert((ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) || (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND original_dest = ra_alloc_ftemp(); IR2_INST * ( * tr_inst_fmadd)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_INST * ( * tr_inst_fmsub)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_OPND temp_add = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); + + la_xvori_b(original_dest, dest, 0); + if (ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd1)) { + src1 = original_dest; + } + if ((ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) && + ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd2)) { + src2 = original_dest; + } switch (op) { case dt_X86_INS_VFMADDSUB132PS: - temp1 = dest, temp2 = src2, temp3 = src1; + temp1 = original_dest, temp2 = src2, temp3 = src1; break; case dt_X86_INS_VFMADDSUB231PS: - temp1 = src1, temp2 = src2, temp3 = dest; + temp1 = src1, temp2 = src2, temp3 = original_dest; break; case dt_X86_INS_VFMADDSUB213PS: - temp1 = src1, temp2 = dest, temp3 = src2; + temp1 = src1, temp2 = original_dest, temp3 = src2; break; default: lsassert(0); @@ -54,8 +65,11 @@ bool translate_vfmaddsubxxxps(IR1_INST * pir1) { la_xvshuf4i_w(temp_add, temp_add, 0xb1); la_xvpackev_w(dest, temp_add, dest); + lasx_fp_fix_fma_nan(dest, temp1, temp2, temp3, false, + ir1_opnd_is_xmm(opnd0) ? 4 : 8); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); + ra_free_temp(original_dest); return true; } @@ -66,26 +80,37 @@ bool translate_vfmaddsubxxxpd(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); + IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); lsassert((ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) || (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND original_dest = ra_alloc_ftemp(); IR2_INST * ( * tr_inst_fmadd)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_INST * ( * tr_inst_fmsub)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_OPND temp_add = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); + + la_xvori_b(original_dest, dest, 0); + if (ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd1)) { + src1 = original_dest; + } + if ((ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) && + ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd2)) { + src2 = original_dest; + } switch (op) { case dt_X86_INS_VFMADDSUB132PD: - temp1 = dest, temp2 = src2, temp3 = src1; + temp1 = original_dest, temp2 = src2, temp3 = src1; break; case dt_X86_INS_VFMADDSUB231PD: - temp1 = src1, temp2 = src2, temp3 = dest; + temp1 = src1, temp2 = src2, temp3 = original_dest; break; case dt_X86_INS_VFMADDSUB213PD: - temp1 = src1, temp2 = dest, temp3 = src2; + temp1 = src1, temp2 = original_dest, temp3 = src2; break; default: lsassert(0); @@ -101,8 +126,11 @@ bool translate_vfmaddsubxxxpd(IR1_INST * pir1) { /* 11 00 11 00 */ /* add[3] sub[2] add[1] sub[0] */ la_xvshuf4i_d(dest, temp_add, 0xc); + lasx_fp_fix_fma_nan(dest, temp1, temp2, temp3, true, + ir1_opnd_is_xmm(opnd0) ? 2 : 4); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); + ra_free_temp(original_dest); return true; } @@ -113,26 +141,37 @@ bool translate_vfmsubaddxxxps(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); + IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); lsassert((ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) || (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND original_dest = ra_alloc_ftemp(); IR2_INST * ( * tr_inst_fmadd)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_INST * ( * tr_inst_fmsub)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_OPND temp_add = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); + + la_xvori_b(original_dest, dest, 0); + if (ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd1)) { + src1 = original_dest; + } + if ((ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) && + ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd2)) { + src2 = original_dest; + } switch (op) { case dt_X86_INS_VFMSUBADD132PS: - temp1 = dest, temp2 = src2, temp3 = src1; + temp1 = original_dest, temp2 = src2, temp3 = src1; break; case dt_X86_INS_VFMSUBADD231PS: - temp1 = src1, temp2 = src2, temp3 = dest; + temp1 = src1, temp2 = src2, temp3 = original_dest; break; case dt_X86_INS_VFMSUBADD213PS: - temp1 = src1, temp2 = dest, temp3 = src2; + temp1 = src1, temp2 = original_dest, temp3 = src2; break; default: lsassert(0); @@ -150,8 +189,11 @@ bool translate_vfmsubaddxxxps(IR1_INST * pir1) { la_xvshuf4i_w(temp_add, temp_add, 0xb1); la_xvpackod_w(dest, dest, temp_add); + lasx_fp_fix_fma_nan(dest, temp1, temp2, temp3, false, + ir1_opnd_is_xmm(opnd0) ? 4 : 8); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); + ra_free_temp(original_dest); return true; } @@ -162,26 +204,37 @@ bool translate_vfmsubaddxxxpd(IR1_INST * pir1) { IR1_OPND * opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND * opnd1 = ir1_get_opnd(pir1, 1); + IR1_OPND * opnd2 = ir1_get_opnd(pir1, 2); lsassert((ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) || (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); - IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); + IR2_OPND src2 = load_freg256_from_ir1(opnd2); + IR2_OPND original_dest = ra_alloc_ftemp(); IR2_INST * ( * tr_inst_fmadd)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_INST * ( * tr_inst_fmsub)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); IR2_OPND temp_add = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); + + la_xvori_b(original_dest, dest, 0); + if (ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd1)) { + src1 = original_dest; + } + if ((ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) && + ir1_opnd_base_reg_num(opnd0) == ir1_opnd_base_reg_num(opnd2)) { + src2 = original_dest; + } switch (op) { case dt_X86_INS_VFMSUBADD132PD: - temp1 = dest, temp2 = src2, temp3 = src1; + temp1 = original_dest, temp2 = src2, temp3 = src1; break; case dt_X86_INS_VFMSUBADD231PD: - temp1 = src1, temp2 = src2, temp3 = dest; + temp1 = src1, temp2 = src2, temp3 = original_dest; break; case dt_X86_INS_VFMSUBADD213PD: - temp1 = src1, temp2 = dest, temp3 = src2; + temp1 = src1, temp2 = original_dest, temp3 = src2; break; default: lsassert(0); @@ -197,8 +250,11 @@ bool translate_vfmsubaddxxxpd(IR1_INST * pir1) { /* 01 10 01 10 */ /* sub[3] add[2] sub[1] add[0] */ la_xvshuf4i_d(dest, temp_add, 0x6); + lasx_fp_fix_fma_nan(dest, temp1, temp2, temp3, true, + ir1_opnd_is_xmm(opnd0) ? 2 : 4); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); + ra_free_temp(original_dest); return true; } #endif From ab3c31ee9ab9cc2756b8f499dedae73f151872cc Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:27:39 +0800 Subject: [PATCH 08/10] latx: repair VADDSUB NaNs with bounded temporary lifetimes Repair addition and subtraction lanes using their preserved original operands before reassembling the result. Introduce the final temporary-lifetime arrangement directly, rather than adding and later repairing the higher-pressure implementation. VADDSUB portions reorganized from d0b2f2a0c5, 8586113675, 4aad1fab5d and f034607870. Final code matches ef9dde911e; runtime alias/NaN testing remains pending. --- target/i386/latx/translator/tr-avx.c | 63 +++++++++++++++++++++------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 1569a8b29be..96ab840fc27 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -750,26 +750,42 @@ bool translate_vaddsubpd(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND add_src1 = ra_alloc_ftemp(); - IR2_OPND add_src2 = ra_alloc_ftemp(); IR2_OPND sub_src1 = ra_alloc_ftemp(); IR2_OPND sub_src2 = ra_alloc_ftemp(); + IR2_OPND add_src1 = ra_alloc_ftemp(); + IR2_OPND add_src2 = ra_alloc_ftemp(); la_xvpackev_d(sub_src1, src1, src1); la_xvpackev_d(sub_src2, src2, src2); la_xvpackod_d(add_src1, src1, src1); la_xvpackod_d(add_src2, src2, src2); + ra_free_temp_auto(src2); + ra_free_temp_auto(src1); if (ir1_opnd_is_xmm(opnd0)) { - la_vfsub_d(sub_src1, sub_src1, sub_src2); - la_vfadd_d(add_src1, add_src1, add_src2); + la_vfsub_d(dest, sub_src1, sub_src2); } else { - la_xvfsub_d(sub_src1, sub_src1, sub_src2); - la_xvfadd_d(add_src1, add_src1, add_src2); + la_xvfsub_d(dest, sub_src1, sub_src2); } - la_xvpackev_d(dest, add_src1, sub_src1); + lasx_fp_fix_binary_nan(dest, sub_src1, sub_src2, true, + ir1_opnd_is_xmm(opnd0) ? 2 : 4); + ra_free_temp(sub_src2); + ra_free_temp(sub_src1); + + IR2_OPND add_result = ra_alloc_ftemp(); + if (ir1_opnd_is_xmm(opnd0)) { + la_vfadd_d(add_result, add_src1, add_src2); + } else { + la_xvfadd_d(add_result, add_src1, add_src2); + } + lasx_fp_fix_binary_nan(add_result, add_src1, add_src2, true, + ir1_opnd_is_xmm(opnd0) ? 2 : 4); + la_xvpackev_d(dest, add_result, dest); if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); } + ra_free_temp(add_result); + ra_free_temp(add_src2); + ra_free_temp(add_src1); return true; } @@ -786,27 +802,42 @@ bool translate_vaddsubps(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - - IR2_OPND add_src1 = ra_alloc_ftemp(); - IR2_OPND add_src2 = ra_alloc_ftemp(); IR2_OPND sub_src1 = ra_alloc_ftemp(); IR2_OPND sub_src2 = ra_alloc_ftemp(); + IR2_OPND add_src1 = ra_alloc_ftemp(); + IR2_OPND add_src2 = ra_alloc_ftemp(); + la_xvpackev_w(sub_src1, src1, src1); la_xvpackev_w(sub_src2, src2, src2); la_xvpackod_w(add_src1, src1, src1); la_xvpackod_w(add_src2, src2, src2); + ra_free_temp_auto(src2); + ra_free_temp_auto(src1); if (ir1_opnd_is_xmm(opnd0)) { - la_vfsub_s(sub_src1, sub_src1, sub_src2); - la_vfadd_s(add_src1, add_src1, add_src2); + la_vfsub_s(dest, sub_src1, sub_src2); } else { - la_xvfsub_s(sub_src1, sub_src1, sub_src2); - la_xvfadd_s(add_src1, add_src1, add_src2); + la_xvfsub_s(dest, sub_src1, sub_src2); } - la_xvpackev_w(dest, add_src1, sub_src1); + lasx_fp_fix_binary_nan(dest, sub_src1, sub_src2, false, + ir1_opnd_is_xmm(opnd0) ? 4 : 8); + ra_free_temp(sub_src2); + ra_free_temp(sub_src1); + + IR2_OPND add_result = ra_alloc_ftemp(); + if (ir1_opnd_is_xmm(opnd0)) { + la_vfadd_s(add_result, add_src1, add_src2); + } else { + la_xvfadd_s(add_result, add_src1, add_src2); + } + lasx_fp_fix_binary_nan(add_result, add_src1, add_src2, false, + ir1_opnd_is_xmm(opnd0) ? 4 : 8); + la_xvpackev_w(dest, add_result, dest); if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); } - + ra_free_temp(add_result); + ra_free_temp(add_src2); + ra_free_temp(add_src1); return true; } From c0e834d5ec5da5fee79ca2e431e09c916754e5a9 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:28:00 +0800 Subject: [PATCH 09/10] latx: repair NaNs after horizontal AVX arithmetic Apply the shared repair to VHADD and VHSUB using the shuffled original inputs at both vector widths. Keep this integration separate from VADDSUB and dot-product reduction. Horizontal-arithmetic portion reorganized from d0b2f2a0c5; final implementation matches ef9dde911e. --- target/i386/latx/translator/tr-avx.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 96ab840fc27..36aec233399 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -862,6 +862,7 @@ bool translate_vhaddpd(IR1_INST * pir1) { la_xvpickev_d(temp1, src2, src1); la_xvpickod_d(temp2, src2, src1); la_xvfadd_d(dest, temp1, temp2); + lasx_fp_fix_binary_nan(dest, temp1, temp2, true, 4); } else { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); @@ -873,6 +874,7 @@ bool translate_vhaddpd(IR1_INST * pir1) { la_vpickev_d(temp1, src2, src1); la_vpickod_d(temp2, src2, src1); la_vfadd_d(temp, temp1, temp2); + lasx_fp_fix_binary_nan(temp, temp1, temp2, true, 2); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -900,6 +902,7 @@ bool translate_vhaddps(IR1_INST * pir1) { la_xvpickev_w(temp1, src2, src1); la_xvpickod_w(temp2, src2, src1); la_xvfadd_s(dest, temp1, temp2); + lasx_fp_fix_binary_nan(dest, temp1, temp2, false, 8); } else { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); @@ -911,6 +914,7 @@ bool translate_vhaddps(IR1_INST * pir1) { la_vpickev_w(temp1, src2, src1); la_vpickod_w(temp2, src2, src1); la_vfadd_s(temp, temp1, temp2); + lasx_fp_fix_binary_nan(temp, temp1, temp2, false, 4); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -938,6 +942,7 @@ bool translate_vhsubpd(IR1_INST * pir1) { la_xvpickev_d(temp1, src2, src1); la_xvpickod_d(temp2, src2, src1); la_xvfsub_d(dest, temp1, temp2); + lasx_fp_fix_binary_nan(dest, temp1, temp2, true, 4); } else { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); @@ -949,6 +954,7 @@ bool translate_vhsubpd(IR1_INST * pir1) { la_vpickev_d(temp1, src2, src1); la_vpickod_d(temp2, src2, src1); la_vfsub_d(temp, temp1, temp2); + lasx_fp_fix_binary_nan(temp, temp1, temp2, true, 2); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } @@ -976,6 +982,7 @@ bool translate_vhsubps(IR1_INST * pir1) { la_xvpickev_w(temp1, src2, src1); la_xvpickod_w(temp2, src2, src1); la_xvfsub_s(dest, temp1, temp2); + lasx_fp_fix_binary_nan(dest, temp1, temp2, false, 8); } else { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); @@ -987,6 +994,7 @@ bool translate_vhsubps(IR1_INST * pir1) { la_vpickev_w(temp1, src2, src1); la_vpickod_w(temp2, src2, src1); la_vfsub_s(temp, temp1, temp2); + lasx_fp_fix_binary_nan(temp, temp1, temp2, false, 4); set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); } From 15b7e515f39afc4019cc7a55ebd172a2c1a46d02 Mon Sep 17 00:00:00 2001 From: Zhaoyang Xie Date: Mon, 7 Sep 2026 00:28:00 +0800 Subject: [PATCH 10/10] latx: preserve NaN priority and output masks in AVX dot products Repair multiplication and reduction results using the original selected inputs. Preserve masked output lanes and use the final in-place VDPPD shuffle strategy to limit vector temporary pressure. Dot-product portions reorganized from d0b2f2a0c5, 4aad1fab5d and f034607870, retaining the classification helper from ef9dde911e. No scalar predecessor or Denormal-input detector is introduced. --- target/i386/latx/translator/tr-avx.c | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 36aec233399..73ccbcad820 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -5139,7 +5139,9 @@ bool translate_vdppd(IR1_INST *pir1) IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); + IR2_OPND temp3 = ra_alloc_ftemp(); uint8_t imm = ir1_opnd_uimm(opnd3); + bool fix_nan = (imm & 0x3) && (imm & 0x30); la_xvxor_v(temp1, temp1, temp1); la_xvxor_v(temp2, temp2, temp2); if(imm & 0x10){ @@ -5150,10 +5152,15 @@ bool translate_vdppd(IR1_INST *pir1) la_xvextrins_d(temp1, src1, 0x11); la_xvextrins_d(temp2, src2, 0x11); } + la_xvori_b(temp3, temp1, 0); if(ir1_opnd_is_xmm(opnd0)) la_vfmul_d(temp1, temp1, temp2); else la_xvfmul_d(temp1, temp1, temp2); + if (fix_nan) { + lasx_fp_fix_binary_nan(temp1, temp3, temp2, true, 2); + } + la_xvori_b(temp3, temp1, 0); la_xvpackod_d(temp2, temp1, temp1); la_xvpackev_d(temp1, temp1, temp1); if(ir1_opnd_is_xmm(opnd0)) @@ -5167,6 +5174,20 @@ bool translate_vdppd(IR1_INST *pir1) if(imm & 0x2){ la_xvextrins_d(dest, temp1, 0x11); } + ra_free_temp(temp1); + ra_free_temp(temp2); + if (fix_nan) { + IR2_OPND sources[] = { temp3, temp3 }; + const int source_shuffles[] = { 0x1, 0x1 }; + + /* Swap in place for the fallback, then swap back for the priority + * source. This avoids a third vector temporary at peak pressure. */ + lasx_fp_fix_vector_nan_from_sources(dest, sources, 2, true, 2, + source_shuffles, true, + (imm & 0x3) != 0x3, + ir2_opnd_new_none()); + } + ra_free_temp(temp3); if(ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); return true; @@ -5188,7 +5209,10 @@ bool translate_vdpps(IR1_INST *pir1) IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); + IR2_OPND temp3 = ra_alloc_ftemp(); uint8_t imm = ir1_opnd_uimm(opnd3); + int lanes = ir1_opnd_is_xmm(opnd0) ? 4 : 8; + bool fix_nan = (imm & 0xf) && (imm & 0xf0); la_xvxor_v(temp1, temp1, temp1); la_xvxor_v(temp2, temp2, temp2); if(imm & 0x10){ @@ -5207,10 +5231,15 @@ bool translate_vdpps(IR1_INST *pir1) la_xvextrins_w(temp1, src1, 0x33); la_xvextrins_w(temp2, src2, 0x33); } + la_xvori_b(temp3, temp1, 0); if(ir1_opnd_is_xmm(opnd0)) la_vfmul_s(temp1, temp1, temp2); else la_xvfmul_s(temp1, temp1, temp2); + if (fix_nan) { + lasx_fp_fix_binary_nan(temp1, temp3, temp2, false, lanes); + } + la_xvori_b(temp3, temp1, 0); la_xvpackod_w(temp2, temp1, temp1); la_xvpackev_w(temp1, temp1, temp1); if(ir1_opnd_is_xmm(opnd0)) @@ -5237,6 +5266,26 @@ bool translate_vdpps(IR1_INST *pir1) if(imm & 0x8){ la_xvextrins_w(dest, temp1, 0x33); } + ra_free_temp(temp1); + if (!fix_nan || (imm & 0xf) == 0xf) { + ra_free_temp(temp2); + } + if (fix_nan) { + IR2_OPND sources[] = { temp3, temp3, temp3, temp3 }; + /* [1,0,3,2], [0,1,2,3], [3,2,1,0], [2,3,0,1] per 128-bit group. */ + const int source_shuffles[] = { 0xb1, -1, 0x1b, 0x4e }; + IR2_OPND preserve_non_nan = (imm & 0xf) == 0xf ? + ir2_opnd_new_none() : temp2; + + lasx_fp_fix_vector_nan_from_sources(dest, sources, 4, false, lanes, + source_shuffles, false, + (imm & 0xf) != 0xf, + preserve_non_nan); + } + if (fix_nan && (imm & 0xf) != 0xf) { + ra_free_temp(temp2); + } + ra_free_temp(temp3); if(ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); return true;