From 308b2418cf98388cc0606caa1e842360b5cbb204 Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 8 Nov 2026 14:07:47 +0000 Subject: [PATCH 1/4] -mm=rc --atomic-rc: reference counts change with atomic operations A retain is a relaxed atomic add; a release is an atomic subtract with release ordering, and the thread that drops the last reference takes an acquire fence before the block is freed. Without the flag the helpers are unchanged. The flag is part of the JIT cache key, and asking for it without -mm=rc is an error. test-compile-atomic-rc: eight threads of a host take and drop references to one object of an rc library; the count afterwards is exactly what it was (without the flag it drifts, 1 -> 7775). test-runner takes -atomic-rc, and six rc programs run with it. Co-Authored-By: Claude Opus 5.5 --- docs/memory-models.md | 9 +- tslang/include/TypeScript/DataStructs.h | 9 ++ .../LowerToLLVM/OwnershipRoutineLogic.h | 85 ++++++++++++++-- tslang/test/tester/CMakeLists.txt | 22 +++++ tslang/test/tester/atomic-rc-host.cpp | 85 ++++++++++++++++ tslang/test/tester/atomic-rc.cmake | 99 +++++++++++++++++++ tslang/test/tester/test-runner.cpp | 9 +- tslang/tslang/jitcache.cpp | 2 +- tslang/tslang/jitnative.cpp | 2 +- tslang/tslang/opts.cpp | 2 + tslang/tslang/tslang.cpp | 8 ++ 11 files changed, 318 insertions(+), 14 deletions(-) create mode 100644 tslang/test/tester/atomic-rc-host.cpp create mode 100644 tslang/test/tester/atomic-rc.cmake diff --git a/docs/memory-models.md b/docs/memory-models.md index 1ae0ec651..27ac2cfdd 100644 --- a/docs/memory-models.md +++ b/docs/memory-models.md @@ -262,7 +262,14 @@ promotion already keep a program's data off the heap, the heap barely matters. rather than being freed twice. The compiler warns when it can see the mismatch. Building everything with the same `-mm=` avoids it. (The standard library is not affected — see above.) -- **Counts are not atomic.** `-mm=rc` is single-threaded today. +- **Counts are not atomic unless you ask.** A plain `-mm=rc` count change is a load, an add and a + store, so two threads taking or dropping references to the same object at once lose updates: the + object is freed while still held, or never. `--atomic-rc` makes every count change one atomic + read-modify-write (relaxed for a retain; release for a release, with an acquire fence before the + last one frees), so objects can be shared between threads. It costs every count change, shared + or not. It covers the modules built with it: code built without it - including a default library + built without it - still changes counts plainly. Keeping the objects themselves consistent is + the program's job (`@atomic`, `Atomics`). ## Shared libraries and `-mm=gc` diff --git a/tslang/include/TypeScript/DataStructs.h b/tslang/include/TypeScript/DataStructs.h index 3e4591fb2..68cdf38fd 100644 --- a/tslang/include/TypeScript/DataStructs.h +++ b/tslang/include/TypeScript/DataStructs.h @@ -102,6 +102,15 @@ struct CompileOptions return memoryModel == MemoryModelRC; } + // --atomic-rc: every reference count change is an atomic read-modify-write, so objects may + // be shared between threads under `rc`. Only `rc` keeps counts; `own` never counts at all. + bool atomicRC = false; + + bool atomicRefCounts() const + { + return isRefCounted() && atomicRC; + } + // Whether the ownership operations are live and lower to code - retains and releases under // `rc`, releases that destroy under `own`. Where only the count itself matters, the // question is isRefCounted(). diff --git a/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h b/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h index 680cd8b81..f68441efe 100644 --- a/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h +++ b/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h @@ -644,13 +644,23 @@ class OwnershipRoutineLogic // A block marked HEAP_BLOCK_IMMORTAL is neither decremented nor ever the last. That is what // lets a string literal, or a `typeof` result pointing into a descriptor, be released like // any other string without writing to read-only memory or freeing a static block. + // + // Under --atomic-rc the decrement is an atomic subtract with release ordering, and the last + // reference is the one that saw the count at 1 before it: two threads dropping the last two + // references at once must not both see 1 left, nor both see 0. The thread that frees then + // takes an acquire fence, so the writes other threads made to the object before giving up + // their references happen before its destruction. The immortal check stays a relaxed read: + // an immortal header is never written again. + // Marking a block immortal (emitMarkImmortal) stays a plain write: it is done to globals and + // keepers before any other thread can reach the block. mlir::Value emitDecRef(mlir::Value payloadPtr) { TypeHelper th(rewriter); auto loc = op->getLoc(); auto parentModule = op->getParentOfType(); - const char *helperName = "__tslang_dec_ref"; + const bool atomic = compileOptions.atomicRefCounts(); + const char *helperName = atomic ? "__tslang_dec_ref_atomic" : "__tslang_dec_ref"; if (!parentModule.lookupSymbol(helperName)) { OpBuilder::InsertionGuard insertGuard(rewriter); @@ -667,7 +677,7 @@ class OwnershipRoutineLogic auto llvmIndexType = tch.convertType(th.getIndexType()); auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, entryBlock->getArgument(0), llvmIndexType); - auto count = rewriter.create(loc, llvmIndexType, blockPtr); + auto count = emitCountLoad(loc, llvmIndexType, blockPtr, atomic); auto immortal = rewriter.create( loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, HEAP_BLOCK_IMMORTAL)); auto isMortal = rewriter.create(loc, LLVM::ICmpPredicate::ne, count, immortal); @@ -680,11 +690,37 @@ class OwnershipRoutineLogic rewriter.setInsertionPointToStart(decBlock); auto one = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); - auto newCount = rewriter.create(loc, llvmIndexType, count, one); - rewriter.create(loc, newCount, blockPtr); - auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); - auto wasLast = rewriter.create(loc, LLVM::ICmpPredicate::eq, newCount, zero); - rewriter.create(loc, ValueRange{wasLast}); + if (atomic) + { + auto oldCount = rewriter.create(loc, LLVM::AtomicBinOp::sub, blockPtr, one, + LLVM::AtomicOrdering::release); + auto wasLast = rewriter.create(loc, LLVM::ICmpPredicate::eq, oldCount, one); + + auto *lastBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *notLastBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + + rewriter.setInsertionPointToEnd(decBlock); + rewriter.create(loc, wasLast, lastBlock, notLastBlock); + + rewriter.setInsertionPointToStart(lastBlock); + rewriter.create(loc, LLVM::AtomicOrdering::acquire); + rewriter.create( + loc, ValueRange{rewriter.create(loc, th.getLLVMBoolType(), + rewriter.getIntegerAttr(th.getLLVMBoolType(), 1))}); + + rewriter.setInsertionPointToStart(notLastBlock); + rewriter.create( + loc, ValueRange{rewriter.create(loc, th.getLLVMBoolType(), + rewriter.getIntegerAttr(th.getLLVMBoolType(), 0))}); + } + else + { + auto newCount = rewriter.create(loc, llvmIndexType, count, one); + rewriter.create(loc, newCount, blockPtr); + auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); + auto wasLast = rewriter.create(loc, LLVM::ICmpPredicate::eq, newCount, zero); + rewriter.create(loc, ValueRange{wasLast}); + } rewriter.setInsertionPointToStart(immortalBlock); rewriter.create( @@ -705,13 +741,18 @@ class OwnershipRoutineLogic // Skipping an immortal block is not an optimisation: incrementing HEAP_BLOCK_IMMORTAL // would turn all-ones into zero, and the next release would read that as "last reference" // and free a string literal. + // + // Under --atomic-rc the increment is an atomic add. Relaxed ordering is enough: whoever takes + // a reference already holds one, so the block cannot be freed under it, and nothing else is + // published by taking a reference. void emitIncRef(mlir::Value payloadPtr) { TypeHelper th(rewriter); auto loc = op->getLoc(); auto parentModule = op->getParentOfType(); - const char *helperName = "__tslang_inc_ref"; + const bool atomic = compileOptions.atomicRefCounts(); + const char *helperName = atomic ? "__tslang_inc_ref_atomic" : "__tslang_inc_ref"; if (!parentModule.lookupSymbol(helperName)) { OpBuilder::InsertionGuard insertGuard(rewriter); @@ -741,7 +782,7 @@ class OwnershipRoutineLogic rewriter.setInsertionPointToStart(loadBlock); auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, payload, llvmIndexType); - auto count = rewriter.create(loc, llvmIndexType, blockPtr); + auto count = emitCountLoad(loc, llvmIndexType, blockPtr, atomic); auto immortal = rewriter.create( loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, HEAP_BLOCK_IMMORTAL)); auto isMortal = rewriter.create(loc, LLVM::ICmpPredicate::ne, count, immortal); @@ -749,7 +790,15 @@ class OwnershipRoutineLogic rewriter.setInsertionPointToStart(incBlock); auto one = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); - rewriter.create(loc, rewriter.create(loc, llvmIndexType, count, one), blockPtr); + if (atomic) + { + rewriter.create(loc, LLVM::AtomicBinOp::add, blockPtr, one, + LLVM::AtomicOrdering::monotonic); + } + else + { + rewriter.create(loc, rewriter.create(loc, llvmIndexType, count, one), blockPtr); + } rewriter.create(loc, ValueRange{}, returnBlock); rewriter.setInsertionPointToStart(returnBlock); @@ -762,6 +811,22 @@ class OwnershipRoutineLogic ValueRange{payloadPtr}); } + // The read of a block's count that decides whether it is immortal. Under --atomic-rc other + // threads may be changing the count at the same time, so it is a relaxed atomic load rather + // than a plain one, which would be a data race. + mlir::Value emitCountLoad(mlir::Location loc, mlir::Type llvmIndexType, mlir::Value blockPtr, bool atomic) + { + if (!atomic) + { + return rewriter.create(loc, llvmIndexType, blockPtr); + } + + auto alignment = compileOptions.sizeBits() / 8; + return rewriter.create(loc, llvmIndexType, blockPtr, alignment, /*isVolatile=*/false, + /*isNonTemporal=*/false, /*isInvariant=*/false, + /*isInvariantGroup=*/false, LLVM::AtomicOrdering::monotonic); + } + // Is this block mortal - that is, not HEAP_BLOCK_IMMORTAL? Inline rather than a helper: it // is one load and one compare, and it is the whole of what a release costs under own. mlir::Value emitIsMortal(mlir::Value payloadPtr) diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index 48cbf9163..a0cd306a2 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -3158,6 +3158,28 @@ if (NOT WIN32 OR TSLANG_HAVE_SHARED_GC) -P "${CMAKE_CURRENT_SOURCE_DIR}/foreign-thread-gc.cmake") endif() +# The same programs as a few of the -mm=rc tests above with --atomic-rc: the count changes are atomic +# read-modify-writes and the last reference takes an acquire fence, and a program single-threaded +# or not has to behave the same. +foreach(arc_test 00strings 00array4_push_pop 00any 00owned_closures 00interface 00shared_global_field_argument) + tslang_add_test(NAME test-jit-rc-atomic-${arc_test} COMMAND test-runner -jit -mm=rc -atomic-rc "${PROJECT_SOURCE_DIR}/test/tester/tests/${arc_test}.ts") + tslang_add_test(NAME test-compile-rc-atomic-${arc_test} COMMAND test-runner -mm=rc -atomic-rc "${PROJECT_SOURCE_DIR}/test/tester/tests/${arc_test}.ts") +endforeach() + +# A -mm=rc --atomic-rc shared library called from a host's threads at once - see atomic-rc.cmake. +add_executable(atomic-rc-host atomic-rc-host.cpp) +if (NOT WIN32) + target_link_libraries(atomic-rc-host PRIVATE ${CMAKE_DL_LIBS} pthread) +endif() +add_test(NAME test-compile-atomic-rc + COMMAND ${CMAKE_COMMAND} + "-DTSLANG=$" + "-DHOST=$" + "-DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}/atomic-rc" + "-DLLVM_LIB=${LLVM_LIBRARY_DIR}" + "-DTSLANG_LIB=${CMAKE_BINARY_DIR}/lib" + -P "${CMAKE_CURRENT_SOURCE_DIR}/atomic-rc.cmake") + # One collector per process with the real default library in it - see defaultlib-collector.cmake. # Skipped (not failed) when no default library has been built; the release workflows run them after # building one. The sources live outside tests/: without the default library they do not compile, diff --git a/tslang/test/tester/atomic-rc-host.cpp b/tslang/test/tester/atomic-rc-host.cpp new file mode 100644 index 000000000..de6189599 --- /dev/null +++ b/tslang/test/tester/atomic-rc-host.cpp @@ -0,0 +1,85 @@ +// A host that loads a `-mm=rc` shared library and calls into it from several threads at once, all +// of them taking and dropping references to the same object. See atomic-rc.cmake. +// +// atomic-rc-host +// +// Each thread calls churn(calls), which takes and drops references to the library's one +// shared object. Afterwards the object's count has to be what it was before. Exits 0 when it is, +// 2 when it is not. + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +using ChurnFn = double (*)(double); +using CountFn = double (*)(); + +int main(int argc, char **argv) +{ + if (argc < 4) + { + std::printf("usage: %s \n", argv[0]); + return 1; + } + + auto threads = std::atoi(argv[2]); + auto calls = static_cast(std::atoi(argv[3])); + +#ifdef _WIN32 + auto library = LoadLibraryExA(argv[1], nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); + if (!library) + { + std::printf("LoadLibrary '%s' failed: %lu\n", argv[1], GetLastError()); + return 1; + } + + auto churn = reinterpret_cast(GetProcAddress(library, "churn")); + auto count = reinterpret_cast(GetProcAddress(library, "count")); +#else + auto library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!library) + { + std::printf("dlopen: %s\n", dlerror()); + return 1; + } + + auto churn = reinterpret_cast(dlsym(library, "churn")); + auto count = reinterpret_cast(dlsym(library, "count")); +#endif + if (!churn || !count) + { + std::printf("'%s' has no 'churn' or no 'count'\n", argv[1]); + return 1; + } + + auto before = count(); + + std::vector pool; + for (int id = 0; id < threads; id++) + { + pool.emplace_back([&]() { + if (churn(calls) != calls) + { + std::printf("churn read the shared object wrong\n"); + std::fflush(stdout); + std::exit(3); + } + }); + } + + for (auto &thread : pool) + { + thread.join(); + } + + auto after = count(); + std::printf("%d threads, %g calls each: count %g before, %g after\n", threads, calls, before, after); + return after == before ? 0 : 2; +} diff --git a/tslang/test/tester/atomic-rc.cmake b/tslang/test/tester/atomic-rc.cmake new file mode 100644 index 000000000..1977d4d9d --- /dev/null +++ b/tslang/test/tester/atomic-rc.cmake @@ -0,0 +1,99 @@ +# --atomic-rc: a `-mm=rc` shared library called from several threads at once (atomic-rc-host.cpp), +# every one of them taking and dropping references to the same object. +# +# Under plain `rc` a count change is a load, an add and a store, so two threads changing the same +# count at once lose one of the changes: the count drifts, and the object is freed while it is +# still held or never freed at all. --atomic-rc makes each change one atomic read-modify-write. +# With it the count after the threads are done has to be the count before, exactly. +# +# The same library built without the flag is run too, but only reported: losing an update is a +# race, so its absence proves nothing, and a drift there is what the flag is for, not a failure. +# Seeing it drift is what shows this test can see the race at all. + +cmake_minimum_required(VERSION 3.17.3) + +foreach(var TSLANG HOST WORK_DIR TSLANG_LIB LLVM_LIB) + if(NOT DEFINED ${var}) + message(FATAL_ERROR "${var} is required") + endif() +endforeach() + +file(REMOVE_RECURSE "${WORK_DIR}") +file(MAKE_DIRECTORY "${WORK_DIR}") + +set(ENV{TSLANG_LIB_PATH} "") + +file(WRITE "${WORK_DIR}/shared_churn.ts" [=[ +class Node { + v = 1; +} + +class Box { + s: Shared; + + constructor(s: Shared) { + this.s = s; + } +} + +// written once, here, and only read after: the threads share the block, not the global +const G = new Shared(new Node()); + +// every Box takes a reference to G's block and gives it back when it goes +export function churn(n: number): number { + let sum = 0; + for (let i = 0; i < n; i++) { + const box = new Box(G); + sum += box.s.value.v; + } + + return sum; +} + +export function count(): number { + return Shared.count(G); +} +]=]) + +set(libs "--tslang-lib-path=${TSLANG_LIB}" "--llvm-lib-path=${LLVM_LIB}") + +if(WIN32) + set(prefix "") + set(suffix ".dll") + set(pic "") +else() + set(prefix "lib") + set(suffix ".so") + set(pic "-relocation-model=pic") +endif() + +function(build_and_run name expect_exact) + set(library "${WORK_DIR}/${prefix}${name}${suffix}") + execute_process(COMMAND "${TSLANG}" --emit=dll --opt -mm=rc ${ARGN} --no-default-lib ${pic} ${libs} + shared_churn.ts -o "${library}" + WORKING_DIRECTORY "${WORK_DIR}" + OUTPUT_VARIABLE out + ERROR_VARIABLE err + RESULT_VARIABLE status) + if(NOT status EQUAL 0) + message(FATAL_ERROR "--emit=dll ${name}: exit ${status}\n${out}\n${err}") + endif() + + execute_process(COMMAND "${HOST}" "${library}" 8 200000 + WORKING_DIRECTORY "${WORK_DIR}" + OUTPUT_VARIABLE out + ERROR_VARIABLE err + RESULT_VARIABLE status + TIMEOUT 240) + if(expect_exact) + if(NOT status EQUAL 0) + message(FATAL_ERROR "${name}: the count after the threads is not the count before: exit ${status}\n${out}\n${err}") + endif() + message(STATUS "${name}: ${out}") + else() + message(STATUS "${name} (reported only, exit ${status}): ${out}") + endif() +endfunction() + +build_and_run(atomic TRUE --atomic-rc) +build_and_run(plain FALSE) diff --git a/tslang/test/tester/test-runner.cpp b/tslang/test/tester/test-runner.cpp index 0a535511e..1873fee79 100644 --- a/tslang/test/tester/test-runner.cpp +++ b/tslang/test/tester/test-runner.cpp @@ -102,6 +102,7 @@ auto fastMath = false; auto memoryModel = std::string(""); // `-no-stack-promotion`: build with every allocation on the heap, for comparing against the default auto noStackPromotion = false; +auto atomicRC = false; auto tslang_opt_ext = std::string(""); // `-x86`: build and run a 32-bit (i686) program. The target triple goes into tslang_opt rather @@ -157,7 +158,7 @@ void writeLibPathSettings(std::ostream &batFile) // would let whichever runner created it first decide the flags for everyone. std::string optVariantSuffix() { - return std::string(fastMath ? "fm" : "") + memoryModel + (x86 ? "x86" : "") + (noStackPromotion ? "nosp" : ""); + return std::string(fastMath ? "fm" : "") + memoryModel + (x86 ? "x86" : "") + (noStackPromotion ? "nosp" : "") + (atomicRC ? "arc" : ""); } // What tslang links into an executable (exe.cpp): mimalloc, under every memory model but gc. A @@ -821,6 +822,12 @@ void readParams(int argc, char **argv, std::vector &files) // into tslang_opt, like -mm=: every file of a multi-file test is built without it tslang_opt += " --no-stack-promotion"; } + else if (std::string(argv[index]) == "-atomic-rc") + { + atomicRC = true; + // into tslang_opt, like -mm=: every file of a multi-file test counts atomically + tslang_opt += " --atomic-rc"; + } // `-mm=gc` is accepted as well as the three that change behaviour, and it is not a no-op: // it names the default explicitly, which is what anyone comparing the models types. // It gets its own cached script like the others - the suffix is what keeps two runners diff --git a/tslang/tslang/jitcache.cpp b/tslang/tslang/jitcache.cpp index 760b1ec62..68c675b8e 100644 --- a/tslang/tslang/jitcache.cpp +++ b/tslang/tslang/jitcache.cpp @@ -360,7 +360,7 @@ class JitUnitBuilder std::string key; llvm::raw_string_ostream os(key); os << (isMain ? "main " + mainFuncName.getValue() : std::string("import")) << "\n"; - os << "mm=" << compileOptions.memoryModel << " opt=" << enableOpt.getValue() << " opt_level=" << optLevel.getValue() + os << "mm=" << compileOptions.memoryModel << " atomic-rc=" << compileOptions.atomicRC << " opt=" << enableOpt.getValue() << " opt_level=" << optLevel.getValue() << " size_level=" << sizeLevel.getValue() << " di=" << compileOptions.generateDebugInfo << " lldb=" << compileOptions.lldbDebugInfo << " no-default-lib=" << compileOptions.noDefaultLib << " builtins=" << compileOptions.enableBuiltins << " strict-null-checks=" << compileOptions.strictNullChecks diff --git a/tslang/tslang/jitnative.cpp b/tslang/tslang/jitnative.cpp index bb4aef58c..eaa40cb38 100644 --- a/tslang/tslang/jitnative.cpp +++ b/tslang/tslang/jitnative.cpp @@ -221,7 +221,7 @@ int linkJitUnits(const char *argv0, llvm::ArrayRef units, CompileOption } os << "crt=" << (debugCRT ? "debug" : "release") << " di=" << compileOptions.generateDebugInfo - << " mm=" << compileOptions.memoryModel << " no-default-lib=" << compileOptions.noDefaultLib << "\n"; + << " mm=" << compileOptions.memoryModel << " atomic-rc=" << compileOptions.atomicRC << " no-default-lib=" << compileOptions.noDefaultLib << "\n"; os << "default-lib=" << (compileOptions.noDefaultLib ? std::string() : getDefaultLibBinary(/*shared=*/false, compileOptions, tm.getTargetTriple())) << "\n"; diff --git a/tslang/tslang/opts.cpp b/tslang/tslang/opts.cpp index b2c12540b..f296251b1 100644 --- a/tslang/tslang/opts.cpp +++ b/tslang/tslang/opts.cpp @@ -17,6 +17,7 @@ namespace cl = llvm::cl; extern cl::opt inputFilename; extern cl::opt emitAction; extern cl::opt memoryModelOpt; +extern cl::opt atomicRCOpt; extern cl::opt disableWarnings; extern cl::opt generateDebugInfo; extern cl::opt lldbDebugInfo; @@ -47,6 +48,7 @@ CompileOptions prepareOptions() CompileOptions compileOptions; compileOptions.isJit = emitAction.getValue() == Action::RunJIT; compileOptions.memoryModel = memoryModelOpt.getValue(); + compileOptions.atomicRC = atomicRCOpt.getValue(); compileOptions.enableBuiltins = enableBuiltins.getValue(); compileOptions.noDefaultLib = noDefaultLib.getValue(); compileOptions.disableWarnings = disableWarnings.getValue(); diff --git a/tslang/tslang/tslang.cpp b/tslang/tslang/tslang.cpp index 2a22b981b..5fbeb5da4 100644 --- a/tslang/tslang/tslang.cpp +++ b/tslang/tslang/tslang.cpp @@ -145,6 +145,7 @@ cl::opt memoryModelOpt("mm", cl::desc("Memory management of co cl::values(clEnumValN(MemoryModelNone, "none", "no reclamation, leak everything")), cl::values(clEnumValN(MemoryModelOwn, "own", "single ownership inferred at compile time, no counting (in development; ownership that cannot be proven is a compile error)")), cl::init(MemoryModelGC), cl::cat(TypeScriptCompilerCategory)); +cl::opt atomicRCOpt("atomic-rc", cl::desc("-mm=rc: change reference counts with atomic operations, so objects can be shared between threads"), cl::cat(TypeScriptCompilerCategory)); cl::opt disableWarnings("nowarn", cl::desc("Disable Warnings"), cl::cat(TypeScriptCompilerCategory)); cl::opt verifyOwnership("verify-ownership", cl::desc("Check that every slot taking a reference gives it back on every path out of the function, unwind paths included"), cl::cat(TypeScriptCompilerCategory)); cl::opt noStackPromotion("no-stack-promotion", cl::desc("Keep every allocation on the heap; by default one that provably never outlives its function goes on the stack"), cl::cat(TypeScriptCompilerCategory)); @@ -447,6 +448,13 @@ int main(int argc, char **argv) cl::ParseCommandLineOptions(argc, argv, "TypeScript native compiler\n"); + // only `rc` keeps counts for the flag to make atomic + if (atomicRCOpt && memoryModelOpt != MemoryModelRC) + { + llvm::WithColor::error(llvm::errs(), "tslang") << "--atomic-rc needs -mm=rc\n"; + return -1; + } + // Before program arguments existed a second positional was an error; keep it one wherever // nothing would receive it if (!programArgs.empty() && (emitAction != Action::RunJIT || newVSCodeFolder || newCMakeFolder || installDefaultLibCmd)) From c2368ccf2075714b95d6703bd9dbbbf731fb615b Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 8 Nov 2026 15:03:16 +0000 Subject: [PATCH 2/4] A block's header word carries a shared bit; its count changes are atomic The top bit of a counted block's header says other threads may hold references to it. The count helpers test the word once - not negative is a block only this thread holds, which counts as before with a plain load, add and store; negative is immortal or shared, and a shared block takes an atomic add, or an atomic subtract with release ordering and an acquire fence before the last reference frees it. --atomic-rc sends every mortal block down the shared path. Shared.count reads the count without the bit. Nothing sets the bit yet. The ray tracer under -mm=rc: 106 -> 111 ms. Co-Authored-By: Claude Opus 5.5 --- tslang/include/TypeScript/Defines.h | 7 + .../LowerToLLVM/OwnershipRoutineLogic.h | 205 +++++++++++------- tslang/lib/TypeScript/LowerToLLVM.cpp | 7 +- 3 files changed, 142 insertions(+), 77 deletions(-) diff --git a/tslang/include/TypeScript/Defines.h b/tslang/include/TypeScript/Defines.h index a854722c1..9a416444b 100644 --- a/tslang/include/TypeScript/Defines.h +++ b/tslang/include/TypeScript/Defines.h @@ -505,6 +505,13 @@ // ABI break. See docs/reference-counting-evaluation.md sections 9.5 and 9.24. #define HEAP_BLOCK_IMMORTAL -1 +// The top bit of a counted block's header word says the block is shared: other threads may hold +// references to it, so its count changes are atomic (OwnershipRoutineLogic::emitIncRef and +// emitDecRef). The rest of the word is the count, which never grows into the bit. A block's bit +// is set once, by the thread that holds it before any other can reach it, and never cleared. +// HEAP_BLOCK_IMMORTAL has the bit set too, so an immortal test always comes first. It is the +// word's top bit, so the value depends on the word size: 1 << (bits - 1). + // Runtime type descriptor. // // The ANY_TYPE slot of an "any" box, and the UNION_TAG_INDEX slot of a tagged union, both diff --git a/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h b/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h index f68441efe..706e863b4 100644 --- a/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h +++ b/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h @@ -645,29 +645,36 @@ class OwnershipRoutineLogic // lets a string literal, or a `typeof` result pointing into a descriptor, be released like // any other string without writing to read-only memory or freeing a static block. // - // Under --atomic-rc the decrement is an atomic subtract with release ordering, and the last - // reference is the one that saw the count at 1 before it: two threads dropping the last two - // references at once must not both see 1 left, nor both see 0. The thread that frees then - // takes an acquire fence, so the writes other threads made to the object before giving up - // their references happen before its destruction. The immortal check stays a relaxed read: - // an immortal header is never written again. - // Marking a block immortal (emitMarkImmortal) stays a plain write: it is done to globals and - // keepers before any other thread can reach the block. + // A block with the shared bit set (other threads may hold references to it) + // is decremented with an atomic subtract with release ordering, and the last reference is the + // one that saw a count of 1 before it: two threads dropping the last two references at once + // must not both see 1 left, nor both see 0. The thread that frees then takes an acquire fence, + // so the writes other threads made to the object before giving up their references happen + // before its destruction. Any other block - the common case, one thread's own - is a plain + // load, subtract and store, as it always was. Under --atomic-rc every block takes the atomic + // path, shared bit or not. + // + // The first read is a plain load (emitCountLoad says why that is enough). An immortal header + // is never written again, and a block's shared bit is only ever set, by the one thread that + // holds the block before any other can reach it, so neither test can be made stale by another + // thread. Marking a block immortal (emitMarkImmortal) stays a plain + // write: it is done to globals and keepers before any other thread can reach the block. mlir::Value emitDecRef(mlir::Value payloadPtr) { TypeHelper th(rewriter); auto loc = op->getLoc(); auto parentModule = op->getParentOfType(); - const bool atomic = compileOptions.atomicRefCounts(); - const char *helperName = atomic ? "__tslang_dec_ref_atomic" : "__tslang_dec_ref"; + const bool atomicAlways = compileOptions.atomicRefCounts(); + const char *helperName = atomicAlways ? "__tslang_dec_ref_atomic" : "__tslang_dec_ref"; if (!parentModule.lookupSymbol(helperName)) { OpBuilder::InsertionGuard insertGuard(rewriter); rewriter.setInsertionPointToStart(parentModule.getBody()); + auto boolType = th.getLLVMBoolType(); auto helper = rewriter.create( - loc, helperName, th.getFunctionType(th.getLLVMBoolType(), {th.getPtrType()}), LLVM::Linkage::Internal); + loc, helperName, th.getFunctionType(boolType, {th.getPtrType()}), LLVM::Linkage::Internal); auto *entryBlock = helper.addEntryBlock(rewriter); rewriter.setInsertionPointToStart(entryBlock); @@ -677,55 +684,51 @@ class OwnershipRoutineLogic auto llvmIndexType = tch.convertType(th.getIndexType()); auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, entryBlock->getArgument(0), llvmIndexType); - auto count = emitCountLoad(loc, llvmIndexType, blockPtr, atomic); - auto immortal = rewriter.create( - loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, HEAP_BLOCK_IMMORTAL)); - auto isMortal = rewriter.create(loc, LLVM::ICmpPredicate::ne, count, immortal); + auto count = emitCountLoad(loc, llvmIndexType, blockPtr); - auto *decBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); - auto *immortalBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *sharedBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *plainBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *lastBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *falseBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); rewriter.setInsertionPointToEnd(entryBlock); - rewriter.create(loc, isMortal, decBlock, immortalBlock); + emitBranchOnHeaderWord(loc, llvmIndexType, count, atomicAlways, plainBlock, sharedBlock, falseBlock); + + auto one = [&]() { + return rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); + }; + + // shared: the count without the bit was 1 before this thread's subtract + rewriter.setInsertionPointToStart(sharedBlock); + auto oldWord = rewriter.create(loc, LLVM::AtomicBinOp::sub, blockPtr, one(), + LLVM::AtomicOrdering::release); + auto oldCount = emitWithoutSharedBit(loc, llvmIndexType, oldWord); + auto wasLastShared = rewriter.create(loc, LLVM::ICmpPredicate::eq, oldCount, one()); + rewriter.create(loc, wasLastShared, lastBlock, falseBlock); - rewriter.setInsertionPointToStart(decBlock); - auto one = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); - if (atomic) + rewriter.setInsertionPointToStart(lastBlock); + rewriter.create(loc, LLVM::AtomicOrdering::acquire); + rewriter.create( + loc, ValueRange{rewriter.create(loc, boolType, rewriter.getIntegerAttr(boolType, 1))}); + + // this thread's own: the bit is clear, so the word is the count + rewriter.setInsertionPointToStart(plainBlock); + if (atomicAlways) { - auto oldCount = rewriter.create(loc, LLVM::AtomicBinOp::sub, blockPtr, one, - LLVM::AtomicOrdering::release); - auto wasLast = rewriter.create(loc, LLVM::ICmpPredicate::eq, oldCount, one); - - auto *lastBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); - auto *notLastBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); - - rewriter.setInsertionPointToEnd(decBlock); - rewriter.create(loc, wasLast, lastBlock, notLastBlock); - - rewriter.setInsertionPointToStart(lastBlock); - rewriter.create(loc, LLVM::AtomicOrdering::acquire); - rewriter.create( - loc, ValueRange{rewriter.create(loc, th.getLLVMBoolType(), - rewriter.getIntegerAttr(th.getLLVMBoolType(), 1))}); - - rewriter.setInsertionPointToStart(notLastBlock); - rewriter.create( - loc, ValueRange{rewriter.create(loc, th.getLLVMBoolType(), - rewriter.getIntegerAttr(th.getLLVMBoolType(), 0))}); + rewriter.create(loc, ValueRange{}, falseBlock); } else { - auto newCount = rewriter.create(loc, llvmIndexType, count, one); + auto newCount = rewriter.create(loc, llvmIndexType, count, one()); rewriter.create(loc, newCount, blockPtr); auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); auto wasLast = rewriter.create(loc, LLVM::ICmpPredicate::eq, newCount, zero); rewriter.create(loc, ValueRange{wasLast}); } - rewriter.setInsertionPointToStart(immortalBlock); + rewriter.setInsertionPointToStart(falseBlock); rewriter.create( - loc, ValueRange{rewriter.create(loc, th.getLLVMBoolType(), - rewriter.getIntegerAttr(th.getLLVMBoolType(), 0))}); + loc, ValueRange{rewriter.create(loc, boolType, rewriter.getIntegerAttr(boolType, 0))}); dropDebugLocations(helper); } @@ -742,17 +745,18 @@ class OwnershipRoutineLogic // would turn all-ones into zero, and the next release would read that as "last reference" // and free a string literal. // - // Under --atomic-rc the increment is an atomic add. Relaxed ordering is enough: whoever takes - // a reference already holds one, so the block cannot be freed under it, and nothing else is - // published by taking a reference. + // A shared block (or any, under --atomic-rc) takes an atomic add. Relaxed ordering is enough: + // whoever takes a reference already holds one, so the block cannot be freed under it, and + // nothing else is published by taking a reference. The count never reaches the shared bit, + // so the add leaves the bit alone. void emitIncRef(mlir::Value payloadPtr) { TypeHelper th(rewriter); auto loc = op->getLoc(); auto parentModule = op->getParentOfType(); - const bool atomic = compileOptions.atomicRefCounts(); - const char *helperName = atomic ? "__tslang_inc_ref_atomic" : "__tslang_inc_ref"; + const bool atomicAlways = compileOptions.atomicRefCounts(); + const char *helperName = atomicAlways ? "__tslang_inc_ref_atomic" : "__tslang_inc_ref"; if (!parentModule.lookupSymbol(helperName)) { OpBuilder::InsertionGuard insertGuard(rewriter); @@ -774,7 +778,8 @@ class OwnershipRoutineLogic auto isNotNull = rewriter.create(loc, LLVM::ICmpPredicate::ne, payload, nullPtr); auto *loadBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); - auto *incBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *sharedBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *plainBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); auto *returnBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); rewriter.setInsertionPointToEnd(entryBlock); @@ -782,22 +787,23 @@ class OwnershipRoutineLogic rewriter.setInsertionPointToStart(loadBlock); auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, payload, llvmIndexType); - auto count = emitCountLoad(loc, llvmIndexType, blockPtr, atomic); - auto immortal = rewriter.create( - loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, HEAP_BLOCK_IMMORTAL)); - auto isMortal = rewriter.create(loc, LLVM::ICmpPredicate::ne, count, immortal); - rewriter.create(loc, isMortal, incBlock, returnBlock); - - rewriter.setInsertionPointToStart(incBlock); - auto one = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); - if (atomic) - { - rewriter.create(loc, LLVM::AtomicBinOp::add, blockPtr, one, - LLVM::AtomicOrdering::monotonic); - } - else + auto count = emitCountLoad(loc, llvmIndexType, blockPtr); + emitBranchOnHeaderWord(loc, llvmIndexType, count, atomicAlways, plainBlock, sharedBlock, returnBlock); + + auto one = [&]() { + return rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); + }; + + rewriter.setInsertionPointToStart(sharedBlock); + rewriter.create(loc, LLVM::AtomicBinOp::add, blockPtr, one(), + LLVM::AtomicOrdering::monotonic); + rewriter.create(loc, ValueRange{}, returnBlock); + + rewriter.setInsertionPointToStart(plainBlock); + if (!atomicAlways) { - rewriter.create(loc, rewriter.create(loc, llvmIndexType, count, one), blockPtr); + rewriter.create(loc, rewriter.create(loc, llvmIndexType, count, one()), + blockPtr); } rewriter.create(loc, ValueRange{}, returnBlock); @@ -811,20 +817,67 @@ class OwnershipRoutineLogic ValueRange{payloadPtr}); } - // The read of a block's count that decides whether it is immortal. Under --atomic-rc other - // threads may be changing the count at the same time, so it is a relaxed atomic load rather - // than a plain one, which would be a data race. - mlir::Value emitCountLoad(mlir::Location loc, mlir::Type llvmIndexType, mlir::Value blockPtr, bool atomic) + // The read of a block's header word by the count helpers. A plain load, though a shared block's + // word may be changing under other threads' atomic adds and subtracts: all the helpers decide + // from it is which kind of block this is, and that cannot change under them. A word only this + // thread holds is written by this thread alone; a shared block's top bit is never cleared, and + // an immortal word never written, so either reads negative whatever the count does. The count + // of a shared block is never taken from this read - its atomic subtract returns its own. An + // atomic load would say the same, but it keeps LLVM from cancelling a retain against the + // release that follows it on a block this thread holds, which is most of what reference + // counting costs in a hot loop (the ray tracer: 4.5% over the counts before blocks could be + // shared, 7% with an atomic load here). + mlir::Value emitCountLoad(mlir::Location loc, mlir::Type llvmIndexType, mlir::Value blockPtr) { - if (!atomic) + return rewriter.create(loc, llvmIndexType, blockPtr); + } + + // The shared bit of a block's header word: the top bit (Defines.h, beside HEAP_BLOCK_IMMORTAL). + // A count never grows into it, and HEAP_BLOCK_IMMORTAL (all ones) has it set too - which is why + // every helper tests for immortal first. + mlir::Value emitSharedBitConstant(mlir::Location loc, mlir::Type llvmIndexType) + { + auto bits = compileOptions.sizeBits(); + return rewriter.create( + loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, llvm::APInt::getOneBitSet(bits, bits - 1))); + } + + // Ends the current block with the three-way branch of the count helpers on a header word: + // `plainBlock` for a block only this thread holds, `sharedBlock` for a shared one, and + // `immortalBlock` for an immortal one. The shared bit is the word's top bit, which + // HEAP_BLOCK_IMMORTAL has set as well, so one signed test - the word is not negative - is all + // the common case pays, as it paid one test for immortal before blocks could be shared. Under + // --atomic-rc every mortal block is treated as shared. + void emitBranchOnHeaderWord(mlir::Location loc, mlir::Type llvmIndexType, mlir::Value word, bool atomicAlways, + mlir::Block *plainBlock, mlir::Block *sharedBlock, mlir::Block *immortalBlock) + { + auto immortal = rewriter.create( + loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, HEAP_BLOCK_IMMORTAL)); + auto isMortal = rewriter.create(loc, LLVM::ICmpPredicate::ne, word, immortal); + if (atomicAlways) { - return rewriter.create(loc, llvmIndexType, blockPtr); + rewriter.create(loc, isMortal, sharedBlock, immortalBlock); + return; } - auto alignment = compileOptions.sizeBits() / 8; - return rewriter.create(loc, llvmIndexType, blockPtr, alignment, /*isVolatile=*/false, - /*isNonTemporal=*/false, /*isInvariant=*/false, - /*isInvariantGroup=*/false, LLVM::AtomicOrdering::monotonic); + auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); + auto isPlain = rewriter.create(loc, LLVM::ICmpPredicate::sge, word, zero); + + auto *currentBlock = rewriter.getInsertionBlock(); + auto *topBitBlock = rewriter.createBlock(currentBlock->getParent(), std::next(currentBlock->getIterator())); + rewriter.create(loc, isMortal, sharedBlock, immortalBlock); + + rewriter.setInsertionPointToEnd(currentBlock); + rewriter.create(loc, isPlain, plainBlock, topBitBlock); + } + + // The count a header word holds, without the shared bit. + mlir::Value emitWithoutSharedBit(mlir::Location loc, mlir::Type llvmIndexType, mlir::Value word) + { + auto bits = compileOptions.sizeBits(); + auto mask = rewriter.create( + loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, llvm::APInt::getLowBitsSet(bits, bits - 1))); + return rewriter.create(loc, llvmIndexType, word, mask); } // Is this block mortal - that is, not HEAP_BLOCK_IMMORTAL? Inline rather than a helper: it diff --git a/tslang/lib/TypeScript/LowerToLLVM.cpp b/tslang/lib/TypeScript/LowerToLLVM.cpp index a1a4b81c8..c8b0c3608 100644 --- a/tslang/lib/TypeScript/LowerToLLVM.cpp +++ b/tslang/lib/TypeScript/LowerToLLVM.cpp @@ -1372,7 +1372,12 @@ class SharedCountOpLowering : public TsLlvmPattern auto llvmIndexType = tch.convertType(th.getIndexType()); auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, transformed.getShared(), llvmIndexType); - auto count = rewriter.create(loc, llvmIndexType, blockPtr); + auto word = rewriter.create(loc, llvmIndexType, blockPtr); + // the count without the shared bit (Defines.h, beside HEAP_BLOCK_IMMORTAL) + auto bits = tsLlvmContext->compileOptions.sizeBits(); + auto mask = rewriter.create( + loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, llvm::APInt::getLowBitsSet(bits, bits - 1))); + auto count = rewriter.create(loc, llvmIndexType, word, mask); rewriter.replaceOpWithNewOp(op, numberType, count); return success(); } From d0173053782d224e4bcc459121161bb51e697f23 Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 8 Nov 2026 15:03:26 +0000 Subject: [PATCH 3/4] Shared.share(v) and @atomic class mark objects shared between threads Shared.share(v) (ts.MarkShared) sets the shared bit of every counted block v reaches - fields, elements, an array's data block, a closure's capture box and cells, and through a new descriptor slot (TYPE_DESCR_SHARE) whatever an any, an interface or a union holds. A block already shared stops the walk, which ends it on a cycle; an immortal one is left alone. Erased under every model but rc. An @atomic class marks each new object once its constructor has run, and a value stored into one of its fields is marked before the store. A class extending one is @atomic too. The descriptor record grows a slot, so code built before this cannot be mixed with code built after it: the default library has to be rebuilt. test-compile-atomic-rc builds the library three more ways - marked by Shared.share, its root an @atomic class, and marked under --atomic-rc - each exact after eight threads; 00shared_share covers a cycle, a literal and every kind of field on one thread. Co-Authored-By: Claude Opus 5.5 --- docs/memory-models.md | 26 +- tslang/include/TypeScript/Defines.h | 9 +- .../TypeScript/LowerToLLVM/LLVMCodeHelper.h | 7 +- .../LowerToLLVM/OwnershipRoutineLogic.h | 417 ++++++++++++++++++ .../LowerToLLVM/TypeDescriptorLogic.h | 2 +- .../TypeScript/MLIRLogic/MLIRGenStore.h | 25 ++ tslang/include/TypeScript/TypeScriptOps.td | 19 + tslang/lib/TypeScript/LowerToAffineLoops.cpp | 2 +- tslang/lib/TypeScript/LowerToLLVM.cpp | 35 +- tslang/lib/TypeScript/MLIRGenAccessCall.cpp | 16 + tslang/lib/TypeScript/MLIRGenClasses.cpp | 5 + tslang/lib/TypeScript/MLIRGenExpressions.cpp | 30 ++ tslang/lib/TypeScript/MLIRGenImpl.h | 36 ++ .../TypeScript/OwnershipInferenceMoves.cpp | 6 + tslang/lib/TypeScript/StackEscapeFacts.h | 2 + tslang/test/tester/CMakeLists.txt | 12 +- tslang/test/tester/atomic-rc.cmake | 100 ++++- tslang/test/tester/tests/00shared_share.ts | 104 +++++ 18 files changed, 819 insertions(+), 34 deletions(-) create mode 100644 tslang/test/tester/tests/00shared_share.ts diff --git a/docs/memory-models.md b/docs/memory-models.md index 27ac2cfdd..3045b9007 100644 --- a/docs/memory-models.md +++ b/docs/memory-models.md @@ -262,14 +262,24 @@ promotion already keep a program's data off the heap, the heap barely matters. rather than being freed twice. The compiler warns when it can see the mismatch. Building everything with the same `-mm=` avoids it. (The standard library is not affected — see above.) -- **Counts are not atomic unless you ask.** A plain `-mm=rc` count change is a load, an add and a - store, so two threads taking or dropping references to the same object at once lose updates: the - object is freed while still held, or never. `--atomic-rc` makes every count change one atomic - read-modify-write (relaxed for a retain; release for a release, with an acquire fence before the - last one frees), so objects can be shared between threads. It costs every count change, shared - or not. It covers the modules built with it: code built without it - including a default library - built without it - still changes counts plainly. Keeping the objects themselves consistent is - the program's job (`@atomic`, `Atomics`). +- **Sharing objects between threads.** A plain count change is a load, an add and a store, so two + threads taking or dropping references to the same object at once lose updates: the object is + freed while still held, or never. So the top bit of a block's header word marks it *shared*, and + a shared block's count changes atomically (relaxed for a retain; release for a release, with an + acquire fence before the last one frees). A block only one thread holds counts as before; the + test costs the ray tracer about 4.5%. There are three ways to share: + - `Shared.share(v)` marks every block `v` reaches - fields, elements, closures' captures, and + what an `any`, interface or union holds. It marks the graph as it is: a value stored into a + shared object later has to be shared too before another thread can see it. + - An `@atomic class` marks each new object once its constructor has run, and marks every value + stored into one of its fields before the store. A class extending one is `@atomic` too. Not + seen: a store through an interface or a base type that is not `@atomic`, and an element added + to an array a field holds (`obj.items.push(x)`) - share those first. + - `--atomic-rc` treats every block as shared: no marking needed, at about 2.5x the counting cost. + + Keeping the objects themselves consistent is the program's job (`@atomic` variables, `Atomics`). + The share routine sits in every type descriptor, so code built before it - a default library + included - cannot be mixed with code built after it; rebuild the library. ## Shared libraries and `-mm=gc` diff --git a/tslang/include/TypeScript/Defines.h b/tslang/include/TypeScript/Defines.h index 9a416444b..8bd2e9b9c 100644 --- a/tslang/include/TypeScript/Defines.h +++ b/tslang/include/TypeScript/Defines.h @@ -414,6 +414,9 @@ // `@value class C` / `@value interface R`: a type whose values are copied, not shared - no identity, // no vtable, no base (docs/superpowers/specs/2026-10-24-value-types-design.md) #define VALUE_DECORATOR "value" +// `@atomic class`: objects several threads may use at once (ClassInfo::isAtomic). On a variable, +// an atomic load and store (MLIRGenVariables). +#define ATOMIC_DECORATOR "atomic" // `@sharedElements class Map`: a container that keeps what it is given and hands out // borrows of it. Under -mm=own each of its type arguments has to be a value, a string (copied) or // a Shared handle (counted): an object, whose block has one owner, goes through Shared. @@ -541,12 +544,16 @@ // because a tagged union carries its payload inline: copying or dropping one has to retain // or release a value whose type is only known at run time, and the tag is what knows it. #define TYPE_DESCR_RETAIN 3 +// Address of the type's share routine (`Shared.share`, OwnershipRoutineLogic::getOrCreateShareRoutine), +// on the same terms again: it sets the shared bit of every block a value whose type is only known +// at run time reaches. Null under every model but `rc`. +#define TYPE_DESCR_SHARE 4 // The block header, last so that it sits immediately in front of the name bytes. A tag is a // `typeof` result, and `typeof x` can be assigned to a `string` and released like any other // string - so a tag has to look like a payload with an immortal block header, exactly as a // string literal does, on top of being a name preceded by a descriptor. Both reads work off // the same pointer: `tag - sizeof(header)` is the marker, `tag - sizeof(record)` the record. -#define TYPE_DESCR_BLOCK_HEADER 4 +#define TYPE_DESCR_BLOCK_HEADER 5 // Coarse category of the described type. These correspond one-to-one with the names // TypeOfOpHelper::typeOfAsString reports, and are derived from that name so the two cannot diff --git a/tslang/include/TypeScript/LowerToLLVM/LLVMCodeHelper.h b/tslang/include/TypeScript/LowerToLLVM/LLVMCodeHelper.h index 507fb1dd8..a02cf981e 100644 --- a/tslang/include/TypeScript/LowerToLLVM/LLVMCodeHelper.h +++ b/tslang/include/TypeScript/LowerToLLVM/LLVMCodeHelper.h @@ -354,7 +354,7 @@ class LLVMCodeHelper : public LLVMCodeHelperBase // capture box); it is not part of the name the record carries. mlir::Value getOrCreateTypeDescriptorName(mlir::Type type, std::string name, int kind, StringRef releaseRoutineName, StringRef retainRoutineName, - StringRef variant = "", uint32_t layout = 0) + StringRef shareRoutineName, StringRef variant = "", uint32_t layout = 0) { auto loc = op->getLoc(); auto parentModule = op->getParentOfType(); @@ -405,6 +405,11 @@ class LLVMCodeHelper : public LLVMCodeHelperBase ? (mlir::Value)rewriter.create(loc, th.getPtrType()) : (mlir::Value)rewriter.create(loc, th.getPtrType(), retainRoutineName); setStructValue(loc, recordValue, retainValue, TYPE_DESCR_RETAIN); + mlir::Value shareValue = + shareRoutineName.empty() + ? (mlir::Value)rewriter.create(loc, th.getPtrType()) + : (mlir::Value)rewriter.create(loc, th.getPtrType(), shareRoutineName); + setStructValue(loc, recordValue, shareValue, TYPE_DESCR_SHARE); // a tag doubles as a string payload, so what precedes the name has to read as an // immortal block header - see TYPE_DESCR_BLOCK_HEADER setStructValue(loc, recordValue, diff --git a/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h b/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h index 706e863b4..c44ce37ae 100644 --- a/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h +++ b/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h @@ -524,6 +524,148 @@ class OwnershipRoutineLogic }); } + // === Sharing between threads === + // + // `Shared.share(v)` (ts.MarkShared) sets the shared bit (Defines.h, beside HEAP_BLOCK_IMMORTAL) + // in the header of every counted block `v` reaches, after which emitIncRef and emitDecRef + // change those counts atomically. "Reaches" is what the release routine of the same type would + // give back - fields, elements, an array's data block, a closure's capture box and its cells, + // and through a descriptor (TYPE_DESCR_SHARE) whatever an `any`, an interface or a union holds - + // so that no block another thread can reach through `v` is counted plainly. A block that is + // already shared stops the walk below it: what it holds was marked when it was, and that is + // also what ends the walk on a cycle. An immortal block is left alone, and has to be: a + // literal's header may be in read-only memory. + // + // Only `rc` keeps counts on every block; under the other models the op is erased. + + // Symbol name of the share routine for `type`, generating it if needed; empty when the type + // owns no heap memory. Like release, it takes the storage holding a value. + std::string getOrCreateShareRoutine(mlir::Type type) + { + if (!compileOptions.isRefCounted() || !ownsHeapMemory(type)) + { + return {}; + } + + std::stringstream nameStream; + nameStream << "tsshr_" << (size_t)hash_value(type); + auto name = nameStream.str(); + + auto parentModule = op->getParentOfType(); + if (parentModule.lookupSymbol(name)) + { + return name; + } + + TypeHelper th(rewriter); + auto loc = op->getLoc(); + + OpBuilder::InsertionGuard insertGuard(rewriter); + rewriter.setInsertionPointToStart(parentModule.getBody()); + + auto funcOp = rewriter.create( + loc, name, th.getFunctionType(th.getVoidType(), {th.getPtrType()}), LLVM::Linkage::Internal); + + // the symbol exists before the body, for a recursive type, as the release routine's does + auto *entryBlock = funcOp.addEntryBlock(rewriter); + rewriter.setInsertionPointToStart(entryBlock); + + buildShareBody(type, entryBlock->getArgument(0)); + + rewriter.create(loc, ValueRange{}); + + dropDebugLocations(funcOp); + return name; + } + + // The share routine of a closure's capture box: the box, each captured variable's cell and what + // the cell holds, and what each field captured by value holds. Keyed by the capture's + // `ref>`, as the box's release routine is. + std::string getOrCreateCaptureBoxShareRoutine(mlir_ts::RefType captureRefType) + { + if (!compileOptions.isRefCounted()) + { + return {}; + } + + std::stringstream nameStream; + nameStream << "tsshrbox_" << (size_t)hash_value(captureRefType); + auto name = nameStream.str(); + + auto parentModule = op->getParentOfType(); + if (parentModule.lookupSymbol(name)) + { + return name; + } + + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + auto loc = op->getLoc(); + auto ptrTy = th.getPtrType(); + + OpBuilder::InsertionGuard insertGuard(rewriter); + rewriter.setInsertionPointToStart(parentModule.getBody()); + + auto funcOp = rewriter.create(loc, name, th.getFunctionType(th.getVoidType(), {ptrTy}), + LLVM::Linkage::Internal); + + auto *entryBlock = funcOp.addEntryBlock(rewriter); + rewriter.setInsertionPointToStart(entryBlock); + + auto tupleType = captureRefType.getElementType(); + auto captureTupleType = dyn_cast(tupleType); + auto boxValue = rewriter.create(loc, ptrTy, entryBlock->getArgument(0)); + emitMarkBlockShared(boxValue, [&]() { + auto llvmTupleType = tch.convertType(tupleType); + for (auto [index, fieldType] : llvm::enumerate(getFieldTypes(tupleType))) + { + // a @value constructor's `this` is the caller's slot, not a cell (releaseCapturedFields) + if (captureTupleType && + MLIRTypeHelper::isValueConstructorThisCapture(captureTupleType.getFields()[index].id, fieldType)) + { + continue; + } + + auto refFieldType = dyn_cast(fieldType); + if (!refFieldType && !ownsHeapMemory(fieldType)) + { + continue; + } + + auto fieldPtr = rewriter.create(loc, ptrTy, llvmTupleType, boxValue, + ArrayRef{0, (int32_t)index}); + if (refFieldType) + { + // the field holds the cell's address: the cell is a counted block of its own + auto cellValue = rewriter.create(loc, ptrTy, fieldPtr); + emitMarkBlockShared(cellValue, [&]() { shareSlot(refFieldType.getElementType(), cellValue); }); + } + else + { + shareSlot(fieldType, fieldPtr); + } + } + }); + + rewriter.create(loc, ValueRange{}); + + dropDebugLocations(funcOp); + return name; + } + + // ts.MarkShared: the share routine of `type` on `value`. + void emitShareValue(mlir::Type type, mlir::Value value) + { + auto wrapperName = getOrCreateValueWrapper(type, getOrCreateShareRoutine(type), "tsshrv_"); + if (wrapperName.empty()) + { + return; + } + + rewriter.create(op->getLoc(), TypeRange{}, FlatSymbolRefAttr::get(rewriter.getContext(), wrapperName), + ValueRange{value}); + } + private: // Field types of a record-shaped type, empty for anything else. llvm::SmallVector getFieldTypes(mlir::Type type) @@ -1718,6 +1860,281 @@ class OwnershipRoutineLogic rewriter.setInsertionPointToStart(continuationBlock); } + + // Sets the shared bit of one block and then runs `thenBody` - the walk into what the block + // holds - only when this call is what set it: not for null, not for an immortal block, not for + // one already shared. The bit is the word's top bit, which an immortal word has too, so a word + // that is not negative is exactly a block to mark. An atomic `or`: the count is left as it is. + void emitMarkBlockShared(mlir::Value payloadPtr, llvm::function_ref thenBody) + { + emitIfNonNull(payloadPtr, [&]() { + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + LLVMCodeHelperBase ch(op, rewriter, typeConverter, compileOptions); + auto loc = op->getLoc(); + + auto llvmIndexType = tch.convertType(th.getIndexType()); + auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, payloadPtr, llvmIndexType); + auto word = emitCountLoad(loc, llvmIndexType, blockPtr); + auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); + auto isUnmarked = rewriter.create(loc, LLVM::ICmpPredicate::sge, word, zero); + + auto *currentBlock = rewriter.getInsertionBlock(); + auto *continuationBlock = rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint()); + auto *markBlock = rewriter.createBlock(continuationBlock); + + rewriter.setInsertionPointToEnd(markBlock); + rewriter.create(loc, LLVM::AtomicBinOp::_or, blockPtr, + emitSharedBitConstant(loc, llvmIndexType), + LLVM::AtomicOrdering::monotonic); + thenBody(); + rewriter.create(loc, ValueRange{}, continuationBlock); + + rewriter.setInsertionPointToEnd(currentBlock); + rewriter.create(loc, isUnmarked, markBlock, continuationBlock); + rewriter.setInsertionPointToStart(continuationBlock); + }); + } + + // Calls the share routine of `type` on `slotPtr`, if it has one. + void shareSlot(mlir::Type type, mlir::Value slotPtr) + { + auto routineName = getOrCreateShareRoutine(type); + if (routineName.empty()) + { + return; + } + + rewriter.create(op->getLoc(), TypeRange{}, + FlatSymbolRefAttr::get(rewriter.getContext(), routineName), ValueRange{slotPtr}); + } + + // Calls the routine a descriptor holds in `descriptorSlot` (TYPE_DESCR_*) on `valueSlotPtr`, if + // it holds one. `tagValue` must not be null. + void callViaDescriptor(mlir::Value tagValue, mlir::Value valueSlotPtr, int32_t descriptorSlot) + { + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + TypeDescriptorLogic tdl(rewriter, tch, op->getLoc()); + + auto loc = op->getLoc(); + + auto recordPtr = tdl.getRecordPtrFromTag(tagValue); + auto routinePtrSlot = + rewriter.create(loc, th.getPtrType(), + TypeDescriptorLogic::getRecordType(rewriter, tch.convertType(th.getIndexType())), + recordPtr, ArrayRef{0, descriptorSlot}); + auto routine = rewriter.create(loc, th.getPtrType(), routinePtrSlot); + + emitIfNonNull(routine, [&]() { + mlir::SmallVector ops{routine, valueSlotPtr}; + auto callOp = rewriter.create(loc, TypeRange{}, ops); + callOp.getProperties().setOperandSegmentSizes({static_cast(ops.size()), 0}); + callOp.setOpBundleSizes({}); + }); + } + + // The fields of a record-shaped value, `basePtr` addressing the record. Every field the release + // routine would visit, and the ones it leaves to an owner elsewhere too: marking more than + // another thread can reach costs atomic counts, marking less costs a race. + void shareFields(mlir::Type recordType, mlir::Value basePtr) + { + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + + auto loc = op->getLoc(); + auto llvmRecordType = tch.convertType(recordType); + + for (auto [index, fieldType] : llvm::enumerate(getFieldTypes(recordType))) + { + auto routineName = std::string(); + if (auto refFieldType = dyn_cast(fieldType)) + { + // a `ref` to a tuple is a capture box (releaseFields); any other `ref` owns nothing + if (isa(refFieldType.getElementType())) + { + routineName = getOrCreateCaptureBoxShareRoutine(refFieldType); + } + } + else + { + routineName = getOrCreateShareRoutine(fieldType); + } + + if (routineName.empty()) + { + continue; + } + + auto fieldPtr = rewriter.create(loc, th.getPtrType(), llvmRecordType, basePtr, + ArrayRef{0, (int32_t)index}); + rewriter.create(loc, TypeRange{}, FlatSymbolRefAttr::get(rewriter.getContext(), routineName), + ValueRange{fieldPtr}); + } + } + + // The share routine's body, case for case with buildBody's. + void buildShareBody(mlir::Type typeIn, mlir::Value slotPtr) + { + auto type = MLIRTypeHelper::stripValueType(typeIn); + + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + + auto loc = op->getLoc(); + auto ptrTy = th.getPtrType(); + + if (isa(type)) + { + auto strValue = rewriter.create(loc, ptrTy, slotPtr); + emitMarkBlockShared(strValue, [&]() {}); + return; + } + + // the header, then its data block, then the elements [0, length) - each only when the one + // before it was not shared already + if (auto arrayType = dyn_cast(type)) + { + ArrayLayout layout(op, rewriter, typeConverter, compileOptions); + auto header = layout.headerForRead(arrayType, slotPtr); + emitMarkBlockShared(header, [&]() { + auto dataValue = rewriter.create(loc, ptrTy, layout.dataAddress(arrayType, header)); + emitMarkBlockShared(dataValue, [&]() { + auto elementRoutine = getOrCreateShareRoutine(arrayType.getElementType()); + if (elementRoutine.empty()) + { + return; + } + + auto llvmIndexType = tch.convertType(th.getIndexType()); + auto sizeValue = rewriter.create(loc, llvmIndexType, layout.lengthAddress(arrayType, header)); + emitCountedLoop(sizeValue, [&](mlir::Value index) { + auto llvmElementType = tch.convertType(arrayType.getElementType()); + auto elementPtr = + rewriter.create(loc, ptrTy, llvmElementType, dataValue, ValueRange{index}); + rewriter.create(loc, TypeRange{}, + FlatSymbolRefAttr::get(rewriter.getContext(), elementRoutine), + ValueRange{elementPtr}); + }); + }); + }); + return; + } + + if (isa(type) || isa(type)) + { + auto storageType = isa(type) ? cast(type).getStorageType() + : cast(type).getStorageType(); + + auto instanceValue = rewriter.create(loc, ptrTy, slotPtr); + emitMarkBlockShared(instanceValue, [&]() { shareFields(storageType, instanceValue); }); + return; + } + + if (auto sharedType = dyn_cast(type)) + { + auto handle = rewriter.create(loc, ptrTy, slotPtr); + emitMarkBlockShared(handle, [&]() { shareSlot(sharedType.getElementType(), handle); }); + return; + } + + if (isa(type)) + { + auto boxValue = rewriter.create(loc, ptrTy, slotPtr); + emitMarkBlockShared(boxValue, [&]() { + auto anyStructType = LLVM::LLVMStructType::getLiteral( + rewriter.getContext(), {tch.convertType(th.getIndexType()), ptrTy, th.getI8Type()}, false); + + auto tagSlot = rewriter.create(loc, ptrTy, anyStructType, boxValue, + ArrayRef{0, ANY_TYPE}); + auto tagValue = rewriter.create(loc, ptrTy, tagSlot); + auto dataSlot = rewriter.create(loc, ptrTy, anyStructType, boxValue, + ArrayRef{0, ANY_DATA}); + emitIfNonNull(tagValue, [&]() { callViaDescriptor(tagValue, dataSlot, TYPE_DESCR_SHARE); }); + }); + return; + } + + // { vtable, this }: the tag of what `this` points at is the vtable's first slot + if (isa(type)) + { + auto llvmType = tch.convertType(type); + auto vtableSlot = rewriter.create(loc, ptrTy, llvmType, slotPtr, + ArrayRef{0, DATA_VALUE_INDEX}); + auto vtableValue = rewriter.create(loc, ptrTy, vtableSlot); + auto thisSlot = rewriter.create(loc, ptrTy, llvmType, slotPtr, + ArrayRef{0, THIS_VALUE_INDEX}); + emitIfNonNull(vtableValue, [&]() { + auto tagSlot = rewriter.create(loc, ptrTy, ptrTy, vtableValue, + ArrayRef{INTERFACE_VTABLE_TYPE_SLOT}); + auto tagValue = rewriter.create(loc, ptrTy, tagSlot); + emitIfNonNull(tagValue, [&]() { callViaDescriptor(tagValue, thisSlot, TYPE_DESCR_SHARE); }); + }); + return; + } + + // { func, this, type }: the tag beside `this` says what `this` is - a capture box, or the + // object a method is bound to + if (isa(type) || isa(type)) + { + auto llvmType = tch.convertType(type); + auto tagSlot = rewriter.create(loc, ptrTy, llvmType, slotPtr, + ArrayRef{0, CLOSURE_TYPE_INDEX}); + auto tagValue = rewriter.create(loc, ptrTy, tagSlot); + auto thisSlot = rewriter.create(loc, ptrTy, llvmType, slotPtr, + ArrayRef{0, THIS_VALUE_INDEX}); + emitIfNonNull(tagValue, [&]() { callViaDescriptor(tagValue, thisSlot, TYPE_DESCR_SHARE); }); + return; + } + + if (auto unionType = dyn_cast(type)) + { + MLIRTypeHelper mth(rewriter.getContext(), compileOptions); + mlir::Type baseType; + if (mth.isUnionTypeNeedsTag(loc, unionType, baseType)) + { + auto llvmUnionType = tch.convertType(unionType); + auto tagSlot = rewriter.create(loc, ptrTy, llvmUnionType, slotPtr, + ArrayRef{0, UNION_TAG_INDEX}); + auto tagValue = rewriter.create(loc, ptrTy, tagSlot); + auto valueSlot = rewriter.create(loc, ptrTy, llvmUnionType, slotPtr, + ArrayRef{0, UNION_VALUE_INDEX}); + emitIfNonNull(tagValue, [&]() { callViaDescriptor(tagValue, valueSlot, TYPE_DESCR_SHARE); }); + } + else + { + shareSlot(baseType, slotPtr); + } + + return; + } + + if (auto optionalType = dyn_cast(type)) + { + auto llvmOptionalType = tch.convertType(optionalType); + auto hasValueSlot = rewriter.create(loc, ptrTy, llvmOptionalType, slotPtr, + ArrayRef{0, OPTIONAL_HASVALUE_INDEX}); + auto hasValue = rewriter.create(loc, th.getLLVMBoolType(), hasValueSlot); + + auto *currentBlock = rewriter.getInsertionBlock(); + auto *continuationBlock = rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint()); + auto *thenBlock = rewriter.createBlock(continuationBlock); + + rewriter.setInsertionPointToEnd(thenBlock); + auto valueSlot = rewriter.create(loc, ptrTy, llvmOptionalType, slotPtr, + ArrayRef{0, OPTIONAL_VALUE_INDEX}); + shareSlot(optionalType.getElementType(), valueSlot); + rewriter.create(loc, ValueRange{}, continuationBlock); + + rewriter.setInsertionPointToEnd(currentBlock); + rewriter.create(loc, hasValue, thenBlock, continuationBlock); + rewriter.setInsertionPointToStart(continuationBlock); + return; + } + + // record-shaped and inline: the value's storage is its holder's, only its fields reach blocks + shareFields(type, slotPtr); + } }; } // namespace typescript diff --git a/tslang/include/TypeScript/LowerToLLVM/TypeDescriptorLogic.h b/tslang/include/TypeScript/LowerToLLVM/TypeDescriptorLogic.h index f0273fcc2..17c380923 100644 --- a/tslang/include/TypeScript/LowerToLLVM/TypeDescriptorLogic.h +++ b/tslang/include/TypeScript/LowerToLLVM/TypeDescriptorLogic.h @@ -37,7 +37,7 @@ class TypeDescriptorLogic { auto i32Ty = builder.getI32Type(); auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext()); - return LLVM::LLVMStructType::getLiteral(builder.getContext(), {i32Ty, i32Ty, ptrTy, ptrTy, llvmIndexType}, false); + return LLVM::LLVMStructType::getLiteral(builder.getContext(), {i32Ty, i32Ty, ptrTy, ptrTy, ptrTy, llvmIndexType}, false); } LLVM::LLVMStructType getRecordType() diff --git a/tslang/include/TypeScript/MLIRLogic/MLIRGenStore.h b/tslang/include/TypeScript/MLIRLogic/MLIRGenStore.h index 9d45cd187..10303a971 100644 --- a/tslang/include/TypeScript/MLIRLogic/MLIRGenStore.h +++ b/tslang/include/TypeScript/MLIRLogic/MLIRGenStore.h @@ -839,6 +839,31 @@ struct ClassInfo // else, a slot left to a declared C++ method null), may have a constructor and field // initializers, and `.new`; still no RTTI. tslang allocates its objects. bool isCppVTableImplementation = false; + // `@atomic`: its objects may be used by several threads at once. Under -mm=rc a new object is + // marked shared, with everything it holds, once its constructor has run, and a value stored + // into one of its fields is marked before the store (ts.MarkShared), so every block reachable + // from it counts atomically. A class extending one is @atomic too (isAtomicClass). Its fields' + // own consistency is still the program's to keep. + bool isAtomic = false; + + // this class or a base class is @atomic + bool isAtomicClass() const + { + if (isAtomic) + { + return true; + } + + for (auto &baseClass : baseClasses) + { + if (baseClass && baseClass->isAtomicClass()) + { + return true; + } + } + + return false; + } // two @cpp methods in one slot, or an override's @slot not its base's: found by // getVirtualTable, which can't report it, and reported by its caller std::string cppSlotConflict; diff --git a/tslang/include/TypeScript/TypeScriptOps.td b/tslang/include/TypeScript/TypeScriptOps.td index ac7a6fd36..978049a2b 100644 --- a/tslang/include/TypeScript/TypeScriptOps.td +++ b/tslang/include/TypeScript/TypeScriptOps.td @@ -571,6 +571,25 @@ def TypeScript_ReleaseOp : TypeScript_Op<"Release"> { let arguments = (ins AnyType:$reference); } +def TypeScript_MarkSharedOp : TypeScript_Op<"MarkShared"> { + let summary = "mark a value, and every block it reaches, shared between threads"; + let description = [{ + `Shared.share(v)`: sets the shared bit (Defines.h, beside HEAP_BLOCK_IMMORTAL) in the header + of every counted block $value reaches - the blocks its release would give back - so that + their count changes are atomic from here on and other threads may take and drop references + to them. A block already shared stops the walk, which is also what ends it on a cycle, and an + immortal one is left alone. Takes no reference and keeps none. + + What is stored into a shared object afterwards is not marked by this: it has to be shared + first (or be stored into a field of an `@atomic` class, which marks what it is given - but + not an element pushed into an array such a field holds). + + Erased under a memory model that keeps no counts. + }]; + + let arguments = (ins AnyType:$value); +} + def TypeScript_RetainSlotOp : TypeScript_Op<"RetainSlot"> { let summary = "take one reference to the value held in a slot"; let description = [{ diff --git a/tslang/lib/TypeScript/LowerToAffineLoops.cpp b/tslang/lib/TypeScript/LowerToAffineLoops.cpp index ee13f9cc6..1254c9f10 100644 --- a/tslang/lib/TypeScript/LowerToAffineLoops.cpp +++ b/tslang/lib/TypeScript/LowerToAffineLoops.cpp @@ -3107,7 +3107,7 @@ void AddTsAffineLegalOps(ConversionTarget &target) mlir_ts::AddressOfOp, mlir_ts::ArithmeticBinaryOp, mlir_ts::ArithmeticUnaryOp, mlir_ts::AssertOp, mlir_ts::CastOp, mlir_ts::ConstantOp, mlir_ts::ElementRefOp, mlir_ts::PointerOffsetRefOp, mlir_ts::FuncOp, mlir_ts::GlobalOp, mlir_ts::GlobalResultOp, mlir_ts::DefaultOp, mlir_ts::HasValueOp, mlir_ts::ValueOp, mlir_ts::ValueOrDefaultOp, mlir_ts::NullOp, mlir_ts::ParseFloatOp, mlir_ts::ParseIntOp, mlir_ts::IsNaNOp, - mlir_ts::PrintOp, mlir_ts::ConvertFOp, mlir_ts::SizeOfOp, mlir_ts::TypeDescriptorOp, mlir_ts::RetainOp, mlir_ts::ReleaseOp, mlir_ts::RetainSlotOp, mlir_ts::ReleaseSlotOp, mlir_ts::RetainCellOp, mlir_ts::ReleaseCellOp, mlir_ts::StoreOp, mlir_ts::SwapOp, mlir_ts::SymbolRefOp, mlir_ts::LengthOfOp, mlir_ts::SetLengthOfOp, + mlir_ts::PrintOp, mlir_ts::ConvertFOp, mlir_ts::SizeOfOp, mlir_ts::TypeDescriptorOp, mlir_ts::RetainOp, mlir_ts::ReleaseOp, mlir_ts::MarkSharedOp, mlir_ts::RetainSlotOp, mlir_ts::ReleaseSlotOp, mlir_ts::RetainCellOp, mlir_ts::ReleaseCellOp, mlir_ts::StoreOp, mlir_ts::SwapOp, mlir_ts::SymbolRefOp, mlir_ts::LengthOfOp, mlir_ts::SetLengthOfOp, mlir_ts::StringLengthOp, mlir_ts::StringResizeOp, mlir_ts::StringConcatOp, mlir_ts::StringCopyOp, mlir_ts::SharedNewOp, mlir_ts::SharedValueRefOp, mlir_ts::SharedCountOp, mlir_ts::StringCompareOp, mlir_ts::AnyCompareOp, mlir_ts::LoadOp, mlir_ts::LoadSaveOp, mlir_ts::NewOp, mlir_ts::CreateTupleOp, mlir_ts::DeconstructTupleOp, mlir_ts::CreateArrayOp, mlir_ts::NewEmptyArrayOp, mlir_ts::NewArrayOp, mlir_ts::DeleteOp, mlir_ts::PropertyRefOp, mlir_ts::InsertPropertyOp, diff --git a/tslang/lib/TypeScript/LowerToLLVM.cpp b/tslang/lib/TypeScript/LowerToLLVM.cpp index c8b0c3608..3ee622a6a 100644 --- a/tslang/lib/TypeScript/LowerToLLVM.cpp +++ b/tslang/lib/TypeScript/LowerToLLVM.cpp @@ -477,10 +477,13 @@ class TypeDescriptorOpLowering : public TsLlvmPattern !MLIRTypeHelper::isSharedHandleType(descriptorType) ? std::string() : orl.getOrCreateRetainRoutine(descriptorType); + // empty but under rc, the one model that counts every block + auto shareRoutineName = orl.getOrCreateShareRoutine(descriptorType); rewriter.replaceOp(op, ch.getOrCreateTypeDescriptorName(descriptorType, name, TypeOfOpHelper::typeKindFromName(name), - releaseRoutineName, retainRoutineName, variant, + releaseRoutineName, retainRoutineName, + shareRoutineName, variant, TypeOfOpHelper::layoutIdOf(descriptorType))); return success(); @@ -520,6 +523,27 @@ class RetainOpLowering : public TsLlvmPattern } }; +// `Shared.share(v)`: the share routine of v's type (OwnershipRoutineLogic::getOrCreateShareRoutine) +// under rc; nothing under the models that do not count every block. +class MarkSharedOpLowering : public TsLlvmPattern +{ + public: + using TsLlvmPattern::TsLlvmPattern; + + LogicalResult matchAndRewrite(mlir_ts::MarkSharedOp op, Adaptor transformed, + ConversionPatternRewriter &rewriter) const final + { + if (tsLlvmContext->compileOptions.isRefCounted()) + { + OwnershipRoutineLogic orl(op, rewriter, getTypeConverter(), tsLlvmContext->compileOptions); + orl.emitShareValue(op.getValue().getType(), transformed.getValue()); + } + + rewriter.eraseOp(op); + return mlir::success(); + } +}; + // A promoted object's fields given back without freeing the object, and with `reset` the object // zeroed - a fresh one where its allocation was. class ReleaseContentsOpLowering : public TsLlvmPattern @@ -7749,10 +7773,12 @@ struct CreateBoundFunctionOpLowering : public TsLlvmPatterncompileOptions.memoryModel == MemoryModelOwn ? std::string() : orl.getOrCreateCaptureBoxRetainRoutine(captureRefType); + auto shareRoutineName = orl.getOrCreateCaptureBoxShareRoutine(captureRefType); LLVMCodeHelper ch(createBoundFunctionOp, rewriter, getTypeConverter(), tsLlvmContext->compileOptions); return ch.getOrCreateTypeDescriptorName(captureRefType, name, TypeOfOpHelper::typeKindFromName(name), - releaseRoutineName, retainRoutineName, borrows ? "b" : ""); + releaseRoutineName, retainRoutineName, shareRoutineName, + borrows ? "b" : ""); } // Under rc, a method bound to a class object counts that object like an interface over it does @@ -7786,10 +7812,11 @@ struct CreateBoundFunctionOpLowering : public TsLlvmPatterncompileOptions); auto releaseRoutineName = orl.getOrCreateReleaseRoutine(thisType); auto retainRoutineName = orl.getOrCreateRetainRoutine(thisType); + auto shareRoutineName = orl.getOrCreateShareRoutine(thisType); LLVMCodeHelper ch(createBoundFunctionOp, rewriter, getTypeConverter(), tsLlvmContext->compileOptions); return ch.getOrCreateTypeDescriptorName(thisType, name, TypeOfOpHelper::typeKindFromName(name), - releaseRoutineName, retainRoutineName); + releaseRoutineName, retainRoutineName, shareRoutineName); } LogicalResult matchAndRewrite(mlir_ts::CreateBoundFunctionOp createBoundFunctionOp, Adaptor transformed, @@ -9720,7 +9747,7 @@ void TypeScriptToLLVMLoweringPass::runOnOperation() ArrayPopOpLowering, ArrayUnshiftOpLowering, ArrayShiftOpLowering, ArraySpliceOpLowering, ArrayViewOpLowering, DeleteOpLowering, ParseFloatOpLowering, ParseIntOpLowering, IsNaNOpLowering, PrintOpLowering, ConvertFOpLowering, StoreOpLowering, SwapOpLowering, SizeOfOpLowering, TypeDescriptorOpLowering, RetainOpLowering, ReleaseOpLowering, RetainSlotOpLowering, ReleaseSlotOpLowering, RetainCellOpLowering, ReleaseCellOpLowering, ReleaseContentsOpLowering, MakeImmortalOpLowering, MarkImmortalCellOpLowering, InsertPropertyOpLowering, LengthOfOpLowering, SetLengthOfOpLowering, StringLengthOpLowering, StringResizeOpLowering, StringConcatOpLowering, StringCopyOpLowering, - SharedNewOpLowering, SharedValueRefOpLowering, SharedCountOpLowering, + SharedNewOpLowering, SharedValueRefOpLowering, SharedCountOpLowering, MarkSharedOpLowering, StringCompareOpLowering, AnyCompareOpLowering, CharToStringOpLowering, UndefOpLowering, CopyStructOpLowering, MemoryCopyOpLowering, MemoryMoveOpLowering, LoadSaveValueLowering, ThrowUnwindOpLowering, ThrowCallOpLowering, VariableOpLowering, DebugVariableOpLowering, AllocaOpLowering, InvokeOpLowering, AccessorUnwindEdgeOpLowering, InvokeHybridOpLowering, VirtualSymbolRefOpLowering, ThisVirtualSymbolRefOpLowering, ThisSymbolRefOpLowering, InterfaceSymbolRefOpLowering, diff --git a/tslang/lib/TypeScript/MLIRGenAccessCall.cpp b/tslang/lib/TypeScript/MLIRGenAccessCall.cpp index bad5d4d71..27a292776 100644 --- a/tslang/lib/TypeScript/MLIRGenAccessCall.cpp +++ b/tslang/lib/TypeScript/MLIRGenAccessCall.cpp @@ -2600,9 +2600,23 @@ namespace mlirgen return mlir::failure(); } + markAtomicInstanceShared(location, classInfo, V(newOp)); return V(newOp); } + // An @atomic class's new object, its constructor done: marked shared with everything its fields + // hold by then (ts.MarkShared). What its constructor stored went through the field barrier + // (markStoreIntoAtomicField) already; this catches what reached a field any other way. Under + // -mm=rc only, the one model whose counts the mark changes: elsewhere it would only keep the + // object off the stack. + void MLIRGenImpl::markAtomicInstanceShared(mlir::Location location, ClassInfo::TypePtr classInfo, mlir::Value instance) + { + if (compileOptions.isRefCounted() && classInfo && classInfo->isAtomicClass() && instance) + { + builder.create(location, instance); + } + } + // `new V(...)` of a @value class: a stack slot for its storage tuple, the constructor run with // `this` pointing at the slot, and the value loaded from it. Nothing is allocated, and nothing // keeps the slot: what `new` gives is a copy. @@ -3104,6 +3118,8 @@ namespace mlirgen auto result = mlirGenCallConstructor(location, classInfo, newOp, operands, false, genContext, constructorName); EXIT_IF_FAILED(result) } + + markAtomicInstanceShared(location, classInfo, newOp); } return newOp; diff --git a/tslang/lib/TypeScript/MLIRGenClasses.cpp b/tslang/lib/TypeScript/MLIRGenClasses.cpp index 9855ad55e..bd82329e3 100644 --- a/tslang/lib/TypeScript/MLIRGenClasses.cpp +++ b/tslang/lib/TypeScript/MLIRGenClasses.cpp @@ -460,6 +460,11 @@ namespace mlirgen { newClassPtr->isValue = true; } + + if (name == ATOMIC_DECORATOR) + { + newClassPtr->isAtomic = true; + } }); getClassesMap().insert({namePtr, newClassPtr}); diff --git a/tslang/lib/TypeScript/MLIRGenExpressions.cpp b/tslang/lib/TypeScript/MLIRGenExpressions.cpp index 82191a919..9bd5b88eb 100644 --- a/tslang/lib/TypeScript/MLIRGenExpressions.cpp +++ b/tslang/lib/TypeScript/MLIRGenExpressions.cpp @@ -1221,6 +1221,15 @@ namespace mlirgen { return mlirGenSharedCount(location, callExpression, genContext); } + + // `Shared.share(v)`: v, and every block it reaches, may be used by other threads + if (access->expression == SyntaxKind::Identifier + && MLIRHelper::getName(access->expression.as()) == "Shared" + && MLIRHelper::getName(access->name) == "share" + && !resolveIdentifier(location, "Shared", genContext)) + { + return mlirGenSharedShare(location, callExpression, genContext); + } } auto result = mlirGen(callExpr, genContext); @@ -1598,6 +1607,27 @@ namespace mlirgen return V(builder.create(location, getNumberType(), handle)); } + // `Shared.share(v)`: sets the shared bit of every counted block `v` reaches (ts.MarkShared), so + // that from here on their counts change atomically and other threads may hold them. Any value: + // one that owns no heap memory has nothing to mark. Returns nothing. + ValueOrLogicalResult MLIRGenImpl::mlirGenSharedShare(mlir::Location location, CallExpression callExpression, + const GenContext &genContext) + { + if (callExpression->arguments.size() != 1) + { + emitError(location, "Shared.share(value) takes one argument"); + return mlir::failure(); + } + + auto result = mlirGen(callExpression->arguments.front(), genContext); + EXIT_IF_FAILED_OR_NO_VALUE(result) + auto value = V(result); + + builder.create(location, value); + + return mlir::success(); + } + ValueOrLogicalResult MLIRGenImpl::mlirGen(NewExpression newExpression, const GenContext &genContext) { auto location = loc(newExpression); diff --git a/tslang/lib/TypeScript/MLIRGenImpl.h b/tslang/lib/TypeScript/MLIRGenImpl.h index e9ca4621f..4f7e5343e 100644 --- a/tslang/lib/TypeScript/MLIRGenImpl.h +++ b/tslang/lib/TypeScript/MLIRGenImpl.h @@ -7005,6 +7005,36 @@ class MLIRGenImpl return mlirGenSaveLogicOneItem(location, fieldValue, savingValue, genContext); } + // The field barrier of an @atomic class: a store through a field of an object whose static type is + // an @atomic class (or extends one) marks the value shared before it lands (ts.MarkShared), so + // nothing another thread can reach through the object is counted plainly. Under -mm=rc only. A + // store through a type that is not @atomic - a base class of one - is not seen. + void markStoreIntoAtomicField(mlir::Location location, mlir::Value storeReference, mlir::Value savingValue) + { + if (!compileOptions.isRefCounted()) + { + return; + } + + auto propertyRefOp = storeReference.getDefiningOp(); + if (!propertyRefOp) + { + return; + } + + auto classType = dyn_cast(propertyRefOp.getObjectRef().getType()); + if (!classType) + { + return; + } + + auto classInfo = getClassInfoByFullName(classType.getName().getValue()); + if (classInfo && classInfo->isAtomicClass()) + { + builder.create(location, savingValue); + } + } + ValueOrLogicalResult mlirGenSaveLogicOneItem(mlir::Location location, mlir::Value leftExpressionValue, mlir::Value rightExpressionValue, const GenContext &genContext) { @@ -7124,6 +7154,10 @@ class MLIRGenImpl return mlir::success(); } + // A field of an @atomic class: what it is given may be read by other threads from here, + // so it is marked shared first (ClassInfo::isAtomic) + markStoreIntoAtomicField(location, storeReference, savingValue); + // Overwriting owning storage hands the count over: the incoming value gains an // owner and the outgoing one loses it. Retaining first is what makes `x = x` safe - // releasing first could drop the last reference and free the value about to be @@ -10895,6 +10929,8 @@ class MLIRGenImpl ValueOrLogicalResult mlirGenNewShared(mlir::Location location, NewExpression newExpression, const GenContext &genContext); ValueOrLogicalResult mlirGenSharedCount(mlir::Location location, CallExpression callExpression, const GenContext &genContext); + ValueOrLogicalResult mlirGenSharedShare(mlir::Location location, CallExpression callExpression, const GenContext &genContext); + void markAtomicInstanceShared(mlir::Location location, ClassInfo::TypePtr classInfo, mlir::Value instance); mlir::LogicalResult mlirGen(DeleteExpression deleteExpression, const GenContext &genContext); diff --git a/tslang/lib/TypeScript/OwnershipInferenceMoves.cpp b/tslang/lib/TypeScript/OwnershipInferenceMoves.cpp index beaaf6533..b0f5df7d1 100644 --- a/tslang/lib/TypeScript/OwnershipInferenceMoves.cpp +++ b/tslang/lib/TypeScript/OwnershipInferenceMoves.cpp @@ -420,6 +420,12 @@ namespace own_inference return true; } + // `Shared.share(v)` sets bits in the headers of what `v` reaches, and keeps nothing + if (mlir::isa(user)) + { + return true; + } + // `--di`'s record of a folded const's value for the debugger: names it, keeps nothing if (mlir::isa(user)) { diff --git a/tslang/lib/TypeScript/StackEscapeFacts.h b/tslang/lib/TypeScript/StackEscapeFacts.h index 9df58e127..ed3f75354 100644 --- a/tslang/lib/TypeScript/StackEscapeFacts.h +++ b/tslang/lib/TypeScript/StackEscapeFacts.h @@ -616,6 +616,8 @@ class StackEscapeFacts return use.getOperandNumber() == 0 ? fieldRefEscapes(elementRefOp.getResult()) : "unclassified"; }) .Case([&](auto) { return ""; }) + // `Shared.share(v)`: other threads may hold it from here, past any frame + .Case([&](auto) { return "shared with other threads"; }) .Case([&](auto) { return ""; }) .Case([&](auto castOp) -> llvm::StringRef { // a view of the same object - a boxed literal seen as an object, a class as its diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index a0cd306a2..afbceb417 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -3161,12 +3161,20 @@ endif() # The same programs as a few of the -mm=rc tests above with --atomic-rc: the count changes are atomic # read-modify-writes and the last reference takes an acquire fence, and a program single-threaded # or not has to behave the same. -foreach(arc_test 00strings 00array4_push_pop 00any 00owned_closures 00interface 00shared_global_field_argument) +foreach(arc_test 00strings 00array4_push_pop 00any 00owned_closures 00interface 00shared_global_field_argument 00shared_share) tslang_add_test(NAME test-jit-rc-atomic-${arc_test} COMMAND test-runner -jit -mm=rc -atomic-rc "${PROJECT_SOURCE_DIR}/test/tester/tests/${arc_test}.ts") tslang_add_test(NAME test-compile-rc-atomic-${arc_test} COMMAND test-runner -mm=rc -atomic-rc "${PROJECT_SOURCE_DIR}/test/tester/tests/${arc_test}.ts") endforeach() -# A -mm=rc --atomic-rc shared library called from a host's threads at once - see atomic-rc.cmake. +# Shared.share and @atomic class on one thread: marking ends on a cycle and leaves a literal alone, +# and a marked block counts and frees as before. Not -mm=own, which rejects the cycle it builds. +foreach(share_mm gc rc none) + tslang_add_test(NAME test-jit-${share_mm}-00-shared-share COMMAND test-runner -jit -mm=${share_mm} "${PROJECT_SOURCE_DIR}/test/tester/tests/00shared_share.ts") + tslang_add_test(NAME test-compile-${share_mm}-00-shared-share COMMAND test-runner -mm=${share_mm} "${PROJECT_SOURCE_DIR}/test/tester/tests/00shared_share.ts") +endforeach() + +# A -mm=rc shared library called from a host's threads at once, its objects shared by --atomic-rc, +# Shared.share or @atomic - see atomic-rc.cmake. add_executable(atomic-rc-host atomic-rc-host.cpp) if (NOT WIN32) target_link_libraries(atomic-rc-host PRIVATE ${CMAKE_DL_LIBS} pthread) diff --git a/tslang/test/tester/atomic-rc.cmake b/tslang/test/tester/atomic-rc.cmake index 1977d4d9d..0d0627659 100644 --- a/tslang/test/tester/atomic-rc.cmake +++ b/tslang/test/tester/atomic-rc.cmake @@ -1,10 +1,13 @@ -# --atomic-rc: a `-mm=rc` shared library called from several threads at once (atomic-rc-host.cpp), -# every one of them taking and dropping references to the same object. +# A `-mm=rc` shared library called from several threads at once (atomic-rc-host.cpp), every one of +# them taking and dropping references to the same objects: built with --atomic-rc, or with the +# objects marked shared by `Shared.share` once at load. # # Under plain `rc` a count change is a load, an add and a store, so two threads changing the same # count at once lose one of the changes: the count drifts, and the object is freed while it is -# still held or never freed at all. --atomic-rc makes each change one atomic read-modify-write. -# With it the count after the threads are done has to be the count before, exactly. +# still held or never freed at all. --atomic-rc makes each change one atomic read-modify-write; +# `Shared.share(v)` sets a bit in the header of every block v reaches, and only those blocks' +# counts change atomically. Either way the count after the threads are done has to be the count +# before, exactly. # # The same library built without the flag is run too, but only reported: losing an update is a # race, so its absence proves nothing, and a drift there is what the flag is for, not a failure. @@ -24,34 +27,80 @@ file(MAKE_DIRECTORY "${WORK_DIR}") set(ENV{TSLANG_LIB_PATH} "") file(WRITE "${WORK_DIR}/shared_churn.ts" [=[ +interface HasV { + v: number; +} + +class Child { + v: number = 1; +} + +function makeReader(c: Child) { + return () => c.v; +} + class Node { - v = 1; + v: number = 1; + // a literal: immortal, which marking has to leave alone (its header may be read-only) + tag = "node"; + child = new Child(); + children: Child[] = [new Child(), new Child()]; + // reached only through a descriptor: what an `any`, an interface, a union and a closure hold + boxed: any = new Child(); + viewed: HasV = new Child(); + either: Child | string = new Child(); + reader = makeReader(new Child()); } class Box { s: Shared; + child: Child; + children: Child[]; + tag: string; + boxed: any; + viewed: HasV; + either: Child | string; + reader: () => number; constructor(s: Shared) { + const node = s.value; this.s = s; + this.child = node.child; + this.children = node.children; + this.tag = node.tag; + this.boxed = node.boxed; + this.viewed = node.viewed; + this.either = node.either; + this.reader = node.reader; } } -// written once, here, and only read after: the threads share the block, not the global -const G = new Shared(new Node()); +// The root every thread starts from: written here, while one thread runs, and only read after. +// Without @atomic or Shared.share its blocks are counted plainly. +ATOMIC_IT class Holder { + s = new Shared(new Node()); + late: Child | undefined; +} + +const H = new Holder(); +// stored after the constructor: an @atomic class's field barrier marks it +H.late = new Child(); +SHARE_IT -// every Box takes a reference to G's block and gives it back when it goes +// every Box takes a reference to each of them, and gives it back when it goes export function churn(n: number): number { - let sum = 0; + let sum: number = 0; for (let i = 0; i < n; i++) { - const box = new Box(G); - sum += box.s.value.v; + const box = new Box(H.s); + const late = H.late; + sum += box.s.value.v * box.child.v * box.children[1].v * box.viewed.v * box.reader() * (late ? late.v : 0); } return sum; } export function count(): number { - return Shared.count(G); + return Shared.count(H.s); } ]=]) @@ -67,10 +116,20 @@ else() set(pic "-relocation-model=pic") endif() -function(build_and_run name expect_exact) +file(READ "${WORK_DIR}/shared_churn.ts" churn_source) +string(REPLACE "ATOMIC_IT" "" churn_source_plain "${churn_source}") +string(REPLACE "SHARE_IT" "" unmarked "${churn_source_plain}") +string(REPLACE "SHARE_IT" "Shared.share(H);" marked "${churn_source_plain}") +string(REPLACE "ATOMIC_IT" "@atomic" churn_source_atomic "${churn_source}") +string(REPLACE "SHARE_IT" "" atomic_class "${churn_source_atomic}") +file(WRITE "${WORK_DIR}/unmarked.ts" "${unmarked}") +file(WRITE "${WORK_DIR}/marked.ts" "${marked}") +file(WRITE "${WORK_DIR}/atomic_class.ts" "${atomic_class}") + +function(build_and_run name source expect_exact) set(library "${WORK_DIR}/${prefix}${name}${suffix}") execute_process(COMMAND "${TSLANG}" --emit=dll --opt -mm=rc ${ARGN} --no-default-lib ${pic} ${libs} - shared_churn.ts -o "${library}" + ${source} -o "${library}" WORKING_DIRECTORY "${WORK_DIR}" OUTPUT_VARIABLE out ERROR_VARIABLE err @@ -95,5 +154,14 @@ function(build_and_run name expect_exact) endif() endfunction() -build_and_run(atomic TRUE --atomic-rc) -build_and_run(plain FALSE) +# every count atomic +build_and_run(atomic unmarked.ts TRUE --atomic-rc) +# Shared.share(H) at load: everything H reaches is marked, and only those counts change atomically +build_and_run(marked marked.ts TRUE) +# `@atomic class Holder`: marked once its constructor has run, and `H.late = ...` marked by the +# field barrier +build_and_run(atomic-class atomic_class.ts TRUE) +# both: a block with the shared bit under --atomic-rc still frees at the right count +build_and_run(marked-atomic marked.ts TRUE --atomic-rc) +# neither: reported only +build_and_run(plain unmarked.ts FALSE) diff --git a/tslang/test/tester/tests/00shared_share.ts b/tslang/test/tester/tests/00shared_share.ts new file mode 100644 index 000000000..3e9888ed2 --- /dev/null +++ b/tslang/test/tester/tests/00shared_share.ts @@ -0,0 +1,104 @@ +// `Shared.share(v)` marks every block v reaches as shared between threads (ts.MarkShared), and an +// `@atomic class` marks its new objects and what its fields are given. On one thread nothing +// changes but how the counts are changed, so this checks that marking ends - on a cycle, on a +// literal whose header may be read-only - and that a marked block still counts and frees as +// before: Shared.count reads the count without the bit, and the program runs to the end under +// rc, where a count the mark disturbed would free a block in use. gc and none count nothing and +// report -1. + +interface HasV { + v: number; +} + +class Leaf { + v: number = 1; +} + +class Ring { + next: Ring | undefined; + name: string; + + constructor(name: string) { + this.name = name; + } +} + +function makeReader(leaf: Leaf) { + return () => leaf.v; +} + +class Bag { + // a literal: immortal, left alone + tag = "bag"; + leaves: Leaf[] = [new Leaf(), new Leaf()]; + names: string[] = ["a", "b"]; + boxed: any = new Leaf(); + viewed: HasV = new Leaf(); + either: Leaf | string = new Leaf(); + maybe: Leaf | undefined = new Leaf(); + reader = makeReader(new Leaf()); + handle = new Shared(new Leaf()); +} + +@atomic +class Counter { + hits: number = 0; + last: Leaf | undefined; + seen: Leaf[] = []; +} + +// a subclass of an @atomic class is @atomic too +class NamedCounter extends Counter { + label: string = "counter"; +} + +function countIs(s: Shared, expected: number) { + const c = Shared.count(s); + return c < 0 || c == expected; +} + +function main() { + // a cycle: the walk stops at the block it marked first + const a = new Ring("a"); + const b = new Ring("b"); + a.next = b; + b.next = a; + Shared.share(a); + Shared.share(a); + assert(a.next.name == "b" && b.next.name == "a", "a marked cycle reads as before"); + b.next = undefined; + + // every kind of field, through descriptors too + const bag = new Bag(); + assert(countIs(bag.handle, 1), "a handle's count before the mark"); + Shared.share(bag); + assert(countIs(bag.handle, 1), "the mark leaves the count alone"); + const extra = bag.handle; + assert(countIs(extra, 2), "a marked handle still counts up"); + assert(bag.leaves[1].v == 1 && bag.names[0] == "a" && bag.viewed.v == 1 && bag.reader() == 1, + "a marked object reads as before"); + assert(bag.tag == "bag", "a literal is left alone"); + + // values that own nothing, and a value marked twice + Shared.share(1); + Shared.share("literal"); + Shared.share(bag.leaves); + Shared.share(bag.leaves); + + // a marked block is still freed: replaced, its last reference goes + bag.leaves = [new Leaf()]; + bag.maybe = undefined; + assert(bag.leaves.length == 1, "a field of a marked object takes a new value"); + + // an @atomic class: marked when made, and each value its fields are given marked as it lands + const counter = new NamedCounter(); + counter.hits = counter.hits + 1; + counter.last = new Leaf(); + // not marked: the barrier sees field stores, not an element added to a field's array. This + // checks only that the program runs on + counter.seen.push(new Leaf()); + counter.last = undefined; + assert(counter.hits == 1 && counter.seen.length == 1 && counter.label == "counter", "an @atomic object reads as before"); + + print("done."); +} From c54f84616cf82ed2af3dc41020a0b059406cfb09 Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 8 Nov 2026 16:04:37 +0000 Subject: [PATCH 4/4] test-compile-atomic-rc checks the default library's DLL too When a default library has been built (DEFAULT_LIB_CANDIDATES, DEFAULT_LIB_PATH), the test also links a library against its DLL whose shared objects are the default library's - a RegExp and strings split made - with RegExp.test running in that DLL on eight threads at once. The descriptor whose share routine marks them is the default library's, so a library built before the share slot would show here. Shared.share, @atomic and both under --atomic-rc keep the count exact; without a default library this part is skipped. --atomic-rc alone is only reported: the default library is built without it, and its own code still counts an unmarked block plainly. memory-models.md says so. Co-Authored-By: Claude Opus 5.5 --- docs/memory-models.md | 3 + tslang/test/tester/CMakeLists.txt | 1 + tslang/test/tester/atomic-rc.cmake | 102 ++++++++++++++++++++++++++++- 3 files changed, 104 insertions(+), 2 deletions(-) diff --git a/docs/memory-models.md b/docs/memory-models.md index 3045b9007..ca4ff9aa8 100644 --- a/docs/memory-models.md +++ b/docs/memory-models.md @@ -276,6 +276,9 @@ promotion already keep a program's data off the heap, the heap barely matters. seen: a store through an interface or a base type that is not `@atomic`, and an element added to an array a field holds (`obj.items.push(x)`) - share those first. - `--atomic-rc` treats every block as shared: no marking needed, at about 2.5x the counting cost. + Only in the code built with it, though: the default library and any other library built + without it still count an unmarked block plainly, so share what their code touches on several + threads. Keeping the objects themselves consistent is the program's job (`@atomic` variables, `Atomics`). The share routine sits in every type descriptor, so code built before it - a default library diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index afbceb417..737fb0e90 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -3186,6 +3186,7 @@ add_test(NAME test-compile-atomic-rc "-DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}/atomic-rc" "-DLLVM_LIB=${LLVM_LIBRARY_DIR}" "-DTSLANG_LIB=${CMAKE_BINARY_DIR}/lib" + "-DDEFAULT_LIB_CANDIDATES=${PROJECT_SOURCE_DIR}/../TypeScriptCompilerDefaultLib/__build;${PROJECT_SOURCE_DIR}/../../TypeScriptCompilerDefaultLib/__build" -P "${CMAKE_CURRENT_SOURCE_DIR}/atomic-rc.cmake") # One collector per process with the real default library in it - see defaultlib-collector.cmake. diff --git a/tslang/test/tester/atomic-rc.cmake b/tslang/test/tester/atomic-rc.cmake index 0d0627659..28d483795 100644 --- a/tslang/test/tester/atomic-rc.cmake +++ b/tslang/test/tester/atomic-rc.cmake @@ -12,10 +12,17 @@ # The same library built without the flag is run too, but only reported: losing an update is a # race, so its absence proves nothing, and a drift there is what the flag is for, not a failure. # Seeing it drift is what shows this test can see the race at all. +# +# Then, when a default library has been built (DEFAULT_LIB_CANDIDATES, DEFAULT_LIB_PATH), a library +# linked against its DLL: the shared objects are a RegExp and strings the default library made, and +# RegExp.test runs in its DLL on all threads at once. The descriptor whose share routine marks them +# is the default library's, so this is where a library built before the share slot would show. +# Under --atomic-rc alone the default library's own code still counts an unmarked block plainly, +# so that build is only reported. cmake_minimum_required(VERSION 3.17.3) -foreach(var TSLANG HOST WORK_DIR TSLANG_LIB LLVM_LIB) +foreach(var TSLANG HOST WORK_DIR TSLANG_LIB LLVM_LIB DEFAULT_LIB_CANDIDATES) if(NOT DEFINED ${var}) message(FATAL_ERROR "${var} is required") endif() @@ -105,6 +112,7 @@ export function count(): number { ]=]) set(libs "--tslang-lib-path=${TSLANG_LIB}" "--llvm-lib-path=${LLVM_LIB}") +set(default_lib_option --no-default-lib) if(WIN32) set(prefix "") @@ -128,7 +136,7 @@ file(WRITE "${WORK_DIR}/atomic_class.ts" "${atomic_class}") function(build_and_run name source expect_exact) set(library "${WORK_DIR}/${prefix}${name}${suffix}") - execute_process(COMMAND "${TSLANG}" --emit=dll --opt -mm=rc ${ARGN} --no-default-lib ${pic} ${libs} + execute_process(COMMAND "${TSLANG}" --emit=dll --opt -mm=rc ${ARGN} ${default_lib_option} ${pic} ${libs} ${source} -o "${library}" WORKING_DIRECTORY "${WORK_DIR}" OUTPUT_VARIABLE out @@ -165,3 +173,93 @@ build_and_run(atomic-class atomic_class.ts TRUE) build_and_run(marked-atomic marked.ts TRUE --atomic-rc) # neither: reported only build_and_run(plain unmarked.ts FALSE) + +# Against the default library's DLL, when one has been built for this target and model +execute_process(COMMAND "${TSLANG}" --print-default-lib-dir=dll --opt -mm=rc + OUTPUT_VARIABLE default_lib_dll_subdir + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE status) +if(NOT status EQUAL 0 OR default_lib_dll_subdir STREQUAL "") + message(FATAL_ERROR "tslang --print-default-lib-dir=dll --opt -mm=rc: exit ${status}") +endif() + +set(default_lib "") +foreach(candidate "$ENV{DEFAULT_LIB_PATH}" ${DEFAULT_LIB_CANDIDATES}) + if(NOT candidate STREQUAL "" AND IS_DIRECTORY "${candidate}/${default_lib_dll_subdir}") + set(default_lib "${candidate}") + break() + endif() +endforeach() + +if(default_lib STREQUAL "") + message(STATUS "no default library built in ${default_lib_dll_subdir}: its DLL is not checked") + return() +endif() + +message(STATUS "default library: ${default_lib}") +set(ENV{DEFAULT_LIB_PATH} "") +set(default_lib_option "--default-lib-path=${default_lib}") +# the host loads a library's dependencies from the library's folder +file(COPY "${default_lib}/${default_lib_dll_subdir}/${prefix}TypeScriptDefaultLib${suffix}" DESTINATION "${WORK_DIR}") + +file(WRITE "${WORK_DIR}/defaultlib_churn.ts" [=[ +class Child { + v: number = 1; +} + +class Box { + s: Shared; + rx: Shared; + words: string[]; + + constructor(s: Shared, rx: Shared, words: string[]) { + this.s = s; + this.rx = rx; + this.words = words; + } +} + +// objects of the default library's: a RegExp, and strings split made +ATOMIC_IT class Holder { + s = new Shared(new Child()); + rx = new Shared(new RegExp("ab")); + words: string[] = "x,ab,y".split(","); +} + +const H = new Holder(); +SHARE_IT + +export function churn(n: number): number { + let sum: number = 0; + for (let i = 0; i < n; i++) { + const box = new Box(H.s, H.rx, H.words); + // in the default library's DLL, on the shared RegExp and a shared string + if (box.rx.value.test(box.words[1])) { + sum += box.s.value.v; + } + } + + return sum; +} + +export function count(): number { + return Shared.count(H.s) * 1000 + Shared.count(H.rx); +} +]=]) + +file(READ "${WORK_DIR}/defaultlib_churn.ts" churn_source) +string(REPLACE "ATOMIC_IT " "" churn_source_plain "${churn_source}") +string(REPLACE "SHARE_IT" "" unmarked "${churn_source_plain}") +string(REPLACE "SHARE_IT" "Shared.share(H);" marked "${churn_source_plain}") +string(REPLACE "ATOMIC_IT " "@atomic " churn_source_atomic "${churn_source}") +string(REPLACE "SHARE_IT" "" atomic_class "${churn_source_atomic}") +file(WRITE "${WORK_DIR}/defaultlib_unmarked.ts" "${unmarked}") +file(WRITE "${WORK_DIR}/defaultlib_marked.ts" "${marked}") +file(WRITE "${WORK_DIR}/defaultlib_atomic_class.ts" "${atomic_class}") + +build_and_run(defaultlib-marked defaultlib_marked.ts TRUE) +build_and_run(defaultlib-atomic-class defaultlib_atomic_class.ts TRUE) +build_and_run(defaultlib-marked-atomic defaultlib_marked.ts TRUE --atomic-rc) +# the default library's own count changes on these blocks stay plain: reported only +build_and_run(defaultlib-atomic defaultlib_unmarked.ts FALSE --atomic-rc) +build_and_run(defaultlib-plain defaultlib_unmarked.ts FALSE)