From 0ca3bd536c641ae09d3ccba79e8672a13ec39750 Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Fri, 13 Nov 2026 10:44:42 +0000 Subject: [PATCH 1/2] wasm: copies and fills of plain loads and stores, not memory.copy and memory.fill The release run for v0.0-pre-alpha126 failed one test: the default library's message_queue.ts under wasm32 -mm=own debug trapped "memory access out of bounds" at the memory.copy of items.shift() in MessageQueue.take. V8 (Node 26, wasm32 and wasm64 alike) checks memory.copy and memory.fill against the size of a shared memory as this thread last saw it. Right after another thread grew the memory - the producer's push reallocating the queue's array into new pages - they trap on that block, while plain loads and stores see the new size at once. A two-thread test of a shared memory reproduces it: memory.copy trapped 112 times in 8000 grows (wasm64: 85), i32.load never. LLVM emits memory.copy and memory.fill for every copy and fill it doesn't expand once bulk-memory is on, which a shared memory's passive data segments need, and has no switch for them alone (WebAssemblySubtarget turns bulk-memory-opt on with bulk-memory). So after the optimizations each llvm.memcpy, memmove and memset of a run-time or large size becomes a call to an internal function of 8-byte and 1-byte loads and stores (__tslang_wasm_memmove, __tslang_wasm_memset). Only the linker's __wasm_init_memory keeps memory.fill, before any thread starts. test-compile-wasm-no-bulk-copy-{wasm32,wasm64} check the assembly; 00wasm_no_bulk_copy.ts runs natively and under the wasm runner. Co-Authored-By: Claude Opus 5.5 --- tslang/test/tester/CMakeLists.txt | 13 ++ .../test/tester/tests/00wasm_no_bulk_copy.ts | 28 ++++ tslang/tslang/obj.cpp | 157 ++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 tslang/test/tester/tests/00wasm_no_bulk_copy.ts diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index da15e98e2..4c50d72e3 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -302,6 +302,17 @@ tslang_add_test(NAME test-compile-00-bound-method-wider-callback COMMAND test-ru tslang_add_test(NAME test-compile-00-optional-union-truthiness COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00optional_union_truthiness.ts") tslang_add_test(NAME test-compile-00-gc-allocations-distinct COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00gc_allocations_distinct.ts") tslang_add_test(NAME test-compile-00-assert-message COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00assert_message.ts") +tslang_add_test(NAME test-compile-00-wasm-no-bulk-copy COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00wasm_no_bulk_copy.ts") +# wasm: a copy or fill of a run-time size is plain loads and stores (V8 checks memory.copy and +# memory.fill against a shared memory's size as this thread last saw it, see obj.cpp) +foreach(wasm_triple wasm32-unknown-unknown wasm64-unknown-unknown) + add_test(NAME test-compile-wasm-no-bulk-copy-${wasm_triple} + COMMAND $ --emit=asm --no-default-lib -mm=none -mtriple=${wasm_triple} + "${PROJECT_SOURCE_DIR}/test/tester/tests/00wasm_no_bulk_copy.ts" -o -) + set_tests_properties(test-compile-wasm-no-bulk-copy-${wasm_triple} + PROPERTIES PASS_REGULAR_EXPRESSION "call[ \t]+__tslang_wasm_memmove" + FAIL_REGULAR_EXPRESSION "memory\\.copy|memory\\.fill|error:|Stack dump|Assertion failed") +endforeach() tslang_add_test(NAME test-compile-01-funcs-vararg COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/01funcs_vararg.ts") # printf is avaliable in JIT only, C has snprintf #tslang_add_test(NAME test-compile-02-funcs-vararg COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/02funcs_vararg.ts") @@ -984,6 +995,7 @@ tslang_add_test(NAME test-jit-00-bound-method-wider-callback COMMAND test-runner tslang_add_test(NAME test-jit-00-optional-union-truthiness COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00optional_union_truthiness.ts") tslang_add_test(NAME test-jit-00-gc-allocations-distinct COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00gc_allocations_distinct.ts") tslang_add_test(NAME test-jit-00-assert-message COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00assert_message.ts") +tslang_add_test(NAME test-jit-00-wasm-no-bulk-copy COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00wasm_no_bulk_copy.ts") tslang_add_test(NAME test-jit-01-funcs-vararg COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/01funcs_vararg.ts") tslang_add_test(NAME test-jit-02-funcs-vararg COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/02funcs_vararg.ts") tslang_add_test(NAME test-jit-00-funcs-bindings COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00funcs_bindings.ts") @@ -2618,6 +2630,7 @@ set(TSLANG_CORPUS 00optional_union_truthiness.ts 00gc_allocations_distinct.ts 00assert_message.ts + 00wasm_no_bulk_copy.ts 00funcs.ts 00generator_manual_next.ts 00generator_manual_next2.ts diff --git a/tslang/test/tester/tests/00wasm_no_bulk_copy.ts b/tslang/test/tester/tests/00wasm_no_bulk_copy.ts new file mode 100644 index 000000000..813b4e4e8 --- /dev/null +++ b/tslang/test/tester/tests/00wasm_no_bulk_copy.ts @@ -0,0 +1,28 @@ +// wasm: a copy or fill whose size is known only at run time is plain loads and stores, not +// memory.copy or memory.fill, which V8 checks against the shared memory's size as this thread last +// saw it (test-compile-wasm-no-bulk-copy-* look at the assembly). Natively it just has to run. +function main() { + const items: number[] = []; + for (let i = 0; i < 100; i++) items.push(i); + + // shift moves the rest down: a copy within the array + assert(items.shift() == 0, "shift gives the first"); + assert(items[0] == 1 && items.length == 99, "shift moves the rest down"); + + // splice moves the tail: a copy to a lower address, then one to a higher address + items.splice(10, 5); + assert(items[10] == 16 && items.length == 94, "splice closes the gap"); + items.splice(10, 0, 100, 101, 102); + assert(items[10] == 100 && items[13] == 16 && items.length == 97, "splice opens a gap"); + + // a copy of an array: a copy between two blocks + const copy = [...items, 200]; + assert(copy.length == 98 && copy[96] == 99 && copy[97] == 200, "the spread copies"); + + // strings: copies of run-time lengths + let text = ""; + for (let i = 0; i < 50; i++) text += `${i},`; + assert(text.length == 140, "the text has every number"); + + print("done."); +} diff --git a/tslang/tslang/obj.cpp b/tslang/tslang/obj.cpp index 8594412b8..bbab4d895 100644 --- a/tslang/tslang/obj.cpp +++ b/tslang/tslang/obj.cpp @@ -5,6 +5,9 @@ // for dump obj #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Verifier.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/InstIterator.h" #include "llvm/CodeGen/CommandFlags.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/MC/TargetRegistry.h" @@ -310,6 +313,154 @@ int setupTargetTriple(llvm::Module *llvmModule, std::unique_ptraddFnAttr(llvm::Attribute::NoUnwind); + fn->addFnAttr("no-builtins"); + + auto *dst = fn->getArg(0); + auto *src = fn->getArg(1); + auto *count = fn->getArg(2); + auto *eight = llvm::ConstantInt::get(sizeTy, 8); + auto *one = llvm::ConstantInt::get(sizeTy, 1); + auto *zero = llvm::ConstantInt::get(sizeTy, 0); + + auto *entry = llvm::BasicBlock::Create(context, "entry", fn); + auto *done = llvm::BasicBlock::Create(context, "done", fn); + llvm::IRBuilder<> builder(done); + builder.CreateRetVoid(); + + // one unit at `at`: 8 bytes or 1, unaligned (wasm loads and stores take any address) + auto unit = [&](llvm::IRBuilder<> &b, llvm::Value *at, llvm::Value *word, bool wide) { + auto *type = wide ? i64Ty : i8Ty; + auto *to = b.CreateGEP(i8Ty, dst, at); + auto *value = fill ? (wide ? word : (llvm::Value *)src) + : (llvm::Value *)b.CreateAlignedLoad(type, b.CreateGEP(i8Ty, src, at), llvm::MaybeAlign(1)); + b.CreateAlignedStore(value, to, llvm::MaybeAlign(1)); + }; + + // up: words then bytes, from the start; down (an overlapping copy to a higher address): from the end + auto loops = [&](llvm::BasicBlock *from, llvm::Value *word, bool up) { + auto *words = llvm::BasicBlock::Create(context, up ? "up.words" : "down.words", fn); + auto *bytesCheck = llvm::BasicBlock::Create(context, up ? "up.bytes.check" : "down.bytes.check", fn); + auto *bytes = llvm::BasicBlock::Create(context, up ? "up.bytes" : "down.bytes", fn); + + llvm::IRBuilder<> b(from); + auto *start = up ? (llvm::Value *)zero : (llvm::Value *)count; + auto *wordsCheck = llvm::BasicBlock::Create(context, up ? "up.words.check" : "down.words.check", fn, words); + b.CreateBr(wordsCheck); + + // words: up while at + 8 <= count, down while at >= 8 + b.SetInsertPoint(wordsCheck); + auto *at = b.CreatePHI(sizeTy, 2); + at->addIncoming(start, from); + auto *more = up ? b.CreateICmpULE(b.CreateAdd(at, eight), count) : b.CreateICmpUGE(at, eight); + b.CreateCondBr(more, words, bytesCheck); + + b.SetInsertPoint(words); + auto *wordAt = up ? (llvm::Value *)at : b.CreateSub(at, eight); + unit(b, wordAt, word, true); + auto *nextWord = up ? b.CreateAdd(at, eight) : wordAt; + at->addIncoming(nextWord, words); + b.CreateBr(wordsCheck); + + // bytes: up while at < count, down while at > 0 + b.SetInsertPoint(bytesCheck); + auto *byteAt = b.CreatePHI(sizeTy, 2); + byteAt->addIncoming(at, wordsCheck); + auto *moreBytes = up ? b.CreateICmpULT(byteAt, count) : b.CreateICmpUGT(byteAt, zero); + b.CreateCondBr(moreBytes, bytes, done); + + b.SetInsertPoint(bytes); + auto *thisByte = up ? (llvm::Value *)byteAt : b.CreateSub(byteAt, one); + unit(b, thisByte, word, false); + byteAt->addIncoming(up ? b.CreateAdd(byteAt, one) : thisByte, bytes); + b.CreateBr(bytesCheck); + }; + + builder.SetInsertPoint(entry); + if (fill) + { + // the byte in each of the word's 8 + auto *word = builder.CreateMul(builder.CreateZExt(src, i64Ty), llvm::ConstantInt::get(i64Ty, 0x0101010101010101ULL)); + auto *body = llvm::BasicBlock::Create(context, "fill", fn, done); + builder.CreateBr(body); + loops(body, word, true); + return fn; + } + + // to a lower address (or no overlap) up from the start, to a higher one down from the end + auto *up = llvm::BasicBlock::Create(context, "up", fn, done); + auto *down = llvm::BasicBlock::Create(context, "down", fn, done); + auto *dstAddress = builder.CreatePtrToInt(dst, sizeTy); + auto *srcAddress = builder.CreatePtrToInt(src, sizeTy); + builder.CreateCondBr(builder.CreateICmpULE(dstAddress, srcAddress), up, down); + loops(up, nullptr, true); + loops(down, nullptr, false); + return fn; +} + +static void replaceWasmBulkMemory(llvm::Module &module) +{ + // the backend expands a constant copy or fill this small (at most 4 stores of 8 bytes, as at -Os) + // into loads and stores + const uint64_t expandedSize = 32; + + llvm::SmallVector calls; + for (auto &function : module) + { + for (auto &instruction : llvm::instructions(function)) + { + if (auto *call = llvm::dyn_cast(&instruction)) + { + auto *size = llvm::dyn_cast(call->getLength()); + if (!size || size->getZExtValue() > expandedSize) + { + calls.push_back(call); + } + } + } + } + + for (auto *call : calls) + { + auto fill = llvm::isa(call); + auto *fn = getWasmMemoryFunction(module, fill); + llvm::IRBuilder<> builder(call); + auto *length = builder.CreateZExtOrTrunc(call->getLength(), fn->getArg(2)->getType()); + auto *second = fill ? llvm::cast(call)->getValue() + : llvm::cast(call)->getSource(); + auto *replacement = builder.CreateCall(fn, {call->getDest(), second, length}); + replacement->setDebugLoc(call->getDebugLoc()); + call->eraseFromParent(); + } +} + int dumpObjOrAssembly(int argc, char **argv, enum Action emitAction, std::string outputFile, mlir::ModuleOp module, CompileOptions &compileOptions) { registerMLIRDialects(module); @@ -349,6 +500,12 @@ int dumpObjOrAssembly(int argc, char **argv, enum Action emitAction, std::string compileOptions.wasmThreads = true; } + // after the optimizations, which would turn the loops back into llvm.memcpy + if (auto triple = llvm::Triple(llvmModule->getTargetTriple()); triple.isWasm() && !triple.isOSEmscripten()) + { + replaceWasmBulkMemory(*llvmModule); + } + // // generate Obj // From b1caad4b4f63e112af29c1f0bb7189a250db239b Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Fri, 13 Nov 2026 11:21:06 +0000 Subject: [PATCH 2/2] wasm: memory.copy and memory.fill after memory.grow 0, not loops The loops of 8-byte and 1-byte loads and stores were too slow for big copies: CI timed json.ts out (defaultlib wasm32 rc and own, wasm64 rc), which builds long strings. memory.size gives the same stale size V8 checks memory.copy against, and memory.grow by 0 brings it up to date: with that check before memory.copy, the two-thread test of a shared memory trapped 0 times in 8000 grows (twice; memory.grow 0 ran 25 and 34 times), where the bare memory.copy trapped 38 times. So __tslang_wasm_memmove and __tslang_wasm_memset now run memory.grow 0 when the copy or fill reaches past memory.size, then the native memory.copy or memory.fill: a load and a compare more than before for the common case. Co-Authored-By: Claude Opus 5.5 --- tslang/test/tester/CMakeLists.txt | 9 +- .../test/tester/tests/00wasm_no_bulk_copy.ts | 6 +- tslang/tslang/obj.cpp | 115 ++++++------------ 3 files changed, 46 insertions(+), 84 deletions(-) diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index 4c50d72e3..0eec523ba 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -303,15 +303,16 @@ tslang_add_test(NAME test-compile-00-optional-union-truthiness COMMAND test-runn tslang_add_test(NAME test-compile-00-gc-allocations-distinct COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00gc_allocations_distinct.ts") tslang_add_test(NAME test-compile-00-assert-message COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00assert_message.ts") tslang_add_test(NAME test-compile-00-wasm-no-bulk-copy COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00wasm_no_bulk_copy.ts") -# wasm: a copy or fill of a run-time size is plain loads and stores (V8 checks memory.copy and -# memory.fill against a shared memory's size as this thread last saw it, see obj.cpp) +# wasm: a copy or fill of a run-time size goes through __tslang_wasm_memmove/memset, whose one +# memory.copy/memory.fill comes after memory.grow 0 when it reaches past memory.size (V8 checks them +# against a shared memory's size as this thread last saw it, see obj.cpp) foreach(wasm_triple wasm32-unknown-unknown wasm64-unknown-unknown) add_test(NAME test-compile-wasm-no-bulk-copy-${wasm_triple} COMMAND $ --emit=asm --no-default-lib -mm=none -mtriple=${wasm_triple} "${PROJECT_SOURCE_DIR}/test/tester/tests/00wasm_no_bulk_copy.ts" -o -) set_tests_properties(test-compile-wasm-no-bulk-copy-${wasm_triple} - PROPERTIES PASS_REGULAR_EXPRESSION "call[ \t]+__tslang_wasm_memmove" - FAIL_REGULAR_EXPRESSION "memory\\.copy|memory\\.fill|error:|Stack dump|Assertion failed") + PROPERTIES PASS_REGULAR_EXPRESSION "call[ \t]+__tslang_wasm_memmove.*memory\\.grow" + FAIL_REGULAR_EXPRESSION "memory\\.copy.*memory\\.copy|memory\\.fill.*memory\\.fill|error:|Stack dump|Assertion failed") endforeach() tslang_add_test(NAME test-compile-01-funcs-vararg COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/01funcs_vararg.ts") # printf is avaliable in JIT only, C has snprintf diff --git a/tslang/test/tester/tests/00wasm_no_bulk_copy.ts b/tslang/test/tester/tests/00wasm_no_bulk_copy.ts index 813b4e4e8..04c1ba0e6 100644 --- a/tslang/test/tester/tests/00wasm_no_bulk_copy.ts +++ b/tslang/test/tester/tests/00wasm_no_bulk_copy.ts @@ -1,6 +1,6 @@ -// wasm: a copy or fill whose size is known only at run time is plain loads and stores, not -// memory.copy or memory.fill, which V8 checks against the shared memory's size as this thread last -// saw it (test-compile-wasm-no-bulk-copy-* look at the assembly). Natively it just has to run. +// wasm: a copy or fill whose size is known only at run time brings the memory's size up to date +// before memory.copy or memory.fill, which V8 checks against the shared memory's size as this thread +// last saw it (test-compile-wasm-no-bulk-copy-* look at the assembly). Natively it just has to run. function main() { const items: number[] = []; for (let i = 0; i < 100; i++) items.push(i); diff --git a/tslang/tslang/obj.cpp b/tslang/tslang/obj.cpp index bbab4d895..a804194b7 100644 --- a/tslang/tslang/obj.cpp +++ b/tslang/tslang/obj.cpp @@ -8,6 +8,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/InstIterator.h" +#include "llvm/IR/IntrinsicsWebAssembly.h" #include "llvm/CodeGen/CommandFlags.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/MC/TargetRegistry.h" @@ -313,17 +314,18 @@ int setupTargetTriple(llvm::Module *llvmModule, std::unique_ptraddFnAttr(llvm::Attribute::NoUnwind); - fn->addFnAttr("no-builtins"); auto *dst = fn->getArg(0); - auto *src = fn->getArg(1); + auto *second = fn->getArg(1); auto *count = fn->getArg(2); - auto *eight = llvm::ConstantInt::get(sizeTy, 8); - auto *one = llvm::ConstantInt::get(sizeTy, 1); - auto *zero = llvm::ConstantInt::get(sizeTy, 0); auto *entry = llvm::BasicBlock::Create(context, "entry", fn); - auto *done = llvm::BasicBlock::Create(context, "done", fn); - llvm::IRBuilder<> builder(done); - builder.CreateRetVoid(); + auto *refresh = llvm::BasicBlock::Create(context, "refresh", fn); + auto *copy = llvm::BasicBlock::Create(context, "copy", fn); + llvm::IRBuilder<> builder(entry); - // one unit at `at`: 8 bytes or 1, unaligned (wasm loads and stores take any address) - auto unit = [&](llvm::IRBuilder<> &b, llvm::Value *at, llvm::Value *word, bool wide) { - auto *type = wide ? i64Ty : i8Ty; - auto *to = b.CreateGEP(i8Ty, dst, at); - auto *value = fill ? (wide ? word : (llvm::Value *)src) - : (llvm::Value *)b.CreateAlignedLoad(type, b.CreateGEP(i8Ty, src, at), llvm::MaybeAlign(1)); - b.CreateAlignedStore(value, to, llvm::MaybeAlign(1)); - }; + // the end of what is touched, in 64 bits (a wasm32 memory's 65536 pages are 2^32 bytes) + auto *last = builder.CreatePtrToInt(dst, sizeTy); + if (!fill) + { + last = builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax, last, builder.CreatePtrToInt(second, sizeTy)); + } - // up: words then bytes, from the start; down (an overlapping copy to a higher address): from the end - auto loops = [&](llvm::BasicBlock *from, llvm::Value *word, bool up) { - auto *words = llvm::BasicBlock::Create(context, up ? "up.words" : "down.words", fn); - auto *bytesCheck = llvm::BasicBlock::Create(context, up ? "up.bytes.check" : "down.bytes.check", fn); - auto *bytes = llvm::BasicBlock::Create(context, up ? "up.bytes" : "down.bytes", fn); - - llvm::IRBuilder<> b(from); - auto *start = up ? (llvm::Value *)zero : (llvm::Value *)count; - auto *wordsCheck = llvm::BasicBlock::Create(context, up ? "up.words.check" : "down.words.check", fn, words); - b.CreateBr(wordsCheck); - - // words: up while at + 8 <= count, down while at >= 8 - b.SetInsertPoint(wordsCheck); - auto *at = b.CreatePHI(sizeTy, 2); - at->addIncoming(start, from); - auto *more = up ? b.CreateICmpULE(b.CreateAdd(at, eight), count) : b.CreateICmpUGE(at, eight); - b.CreateCondBr(more, words, bytesCheck); - - b.SetInsertPoint(words); - auto *wordAt = up ? (llvm::Value *)at : b.CreateSub(at, eight); - unit(b, wordAt, word, true); - auto *nextWord = up ? b.CreateAdd(at, eight) : wordAt; - at->addIncoming(nextWord, words); - b.CreateBr(wordsCheck); - - // bytes: up while at < count, down while at > 0 - b.SetInsertPoint(bytesCheck); - auto *byteAt = b.CreatePHI(sizeTy, 2); - byteAt->addIncoming(at, wordsCheck); - auto *moreBytes = up ? b.CreateICmpULT(byteAt, count) : b.CreateICmpUGT(byteAt, zero); - b.CreateCondBr(moreBytes, bytes, done); - - b.SetInsertPoint(bytes); - auto *thisByte = up ? (llvm::Value *)byteAt : b.CreateSub(byteAt, one); - unit(b, thisByte, word, false); - byteAt->addIncoming(up ? b.CreateAdd(byteAt, one) : thisByte, bytes); - b.CreateBr(bytesCheck); - }; + auto *end = builder.CreateAdd(builder.CreateZExt(last, i64Ty), builder.CreateZExt(count, i64Ty)); + auto *zeroIndex = llvm::ConstantInt::get(i32Ty, 0); + auto *pages = builder.CreateIntrinsic(llvm::Intrinsic::wasm_memory_size, {sizeTy}, {zeroIndex}); + auto *size = builder.CreateShl(builder.CreateZExt(pages, i64Ty), 16); + builder.CreateCondBr(builder.CreateICmpUGT(end, size), refresh, copy); - builder.SetInsertPoint(entry); + builder.SetInsertPoint(refresh); + builder.CreateIntrinsic(llvm::Intrinsic::wasm_memory_grow, {sizeTy}, {zeroIndex, llvm::ConstantInt::get(sizeTy, 0)}); + builder.CreateBr(copy); + + // memory.copy or memory.fill: this function's own call isn't replaced + builder.SetInsertPoint(copy); if (fill) { - // the byte in each of the word's 8 - auto *word = builder.CreateMul(builder.CreateZExt(src, i64Ty), llvm::ConstantInt::get(i64Ty, 0x0101010101010101ULL)); - auto *body = llvm::BasicBlock::Create(context, "fill", fn, done); - builder.CreateBr(body); - loops(body, word, true); - return fn; + builder.CreateMemSet(dst, second, count, llvm::MaybeAlign(1)); + } + else + { + builder.CreateMemMove(dst, llvm::MaybeAlign(1), second, llvm::MaybeAlign(1), count); } - // to a lower address (or no overlap) up from the start, to a higher one down from the end - auto *up = llvm::BasicBlock::Create(context, "up", fn, done); - auto *down = llvm::BasicBlock::Create(context, "down", fn, done); - auto *dstAddress = builder.CreatePtrToInt(dst, sizeTy); - auto *srcAddress = builder.CreatePtrToInt(src, sizeTy); - builder.CreateCondBr(builder.CreateICmpULE(dstAddress, srcAddress), up, down); - loops(up, nullptr, true); - loops(down, nullptr, false); + builder.CreateRetVoid(); return fn; } @@ -500,7 +461,7 @@ int dumpObjOrAssembly(int argc, char **argv, enum Action emitAction, std::string compileOptions.wasmThreads = true; } - // after the optimizations, which would turn the loops back into llvm.memcpy + // after the optimizations, which would inline the calls if (auto triple = llvm::Triple(llvmModule->getTargetTriple()); triple.isWasm() && !triple.isOSEmscripten()) { replaceWasmBulkMemory(*llvmModule);