diff --git a/docs/memory-models.md b/docs/memory-models.md index 1ae0ec651..ca4ff9aa8 100644 --- a/docs/memory-models.md +++ b/docs/memory-models.md @@ -262,7 +262,27 @@ promotion already keep a program's data off the heap, the heap barely matters. rather than being freed twice. The compiler warns when it can see the mismatch. Building everything with the same `-mm=` avoids it. (The standard library is not affected — see above.) -- **Counts are not atomic.** `-mm=rc` is single-threaded today. +- **Sharing objects between threads.** A plain count change is a load, an add and a store, so two + threads taking or dropping references to the same object at once lose updates: the object is + freed while still held, or never. So the top bit of a block's header word marks it *shared*, and + a shared block's count changes atomically (relaxed for a retain; release for a release, with an + acquire fence before the last one frees). A block only one thread holds counts as before; the + test costs the ray tracer about 4.5%. There are three ways to share: + - `Shared.share(v)` marks every block `v` reaches - fields, elements, closures' captures, and + what an `any`, interface or union holds. It marks the graph as it is: a value stored into a + shared object later has to be shared too before another thread can see it. + - An `@atomic class` marks each new object once its constructor has run, and marks every value + stored into one of its fields before the store. A class extending one is `@atomic` too. Not + seen: a store through an interface or a base type that is not `@atomic`, and an element added + to an array a field holds (`obj.items.push(x)`) - share those first. + - `--atomic-rc` treats every block as shared: no marking needed, at about 2.5x the counting cost. + Only in the code built with it, though: the default library and any other library built + without it still count an unmarked block plainly, so share what their code touches on several + threads. + + Keeping the objects themselves consistent is the program's job (`@atomic` variables, `Atomics`). + The share routine sits in every type descriptor, so code built before it - a default library + included - cannot be mixed with code built after it; rebuild the library. ## Shared libraries and `-mm=gc` diff --git a/tslang/include/TypeScript/DataStructs.h b/tslang/include/TypeScript/DataStructs.h index 3e4591fb2..68cdf38fd 100644 --- a/tslang/include/TypeScript/DataStructs.h +++ b/tslang/include/TypeScript/DataStructs.h @@ -102,6 +102,15 @@ struct CompileOptions return memoryModel == MemoryModelRC; } + // --atomic-rc: every reference count change is an atomic read-modify-write, so objects may + // be shared between threads under `rc`. Only `rc` keeps counts; `own` never counts at all. + bool atomicRC = false; + + bool atomicRefCounts() const + { + return isRefCounted() && atomicRC; + } + // Whether the ownership operations are live and lower to code - retains and releases under // `rc`, releases that destroy under `own`. Where only the count itself matters, the // question is isRefCounted(). diff --git a/tslang/include/TypeScript/Defines.h b/tslang/include/TypeScript/Defines.h index a854722c1..8bd2e9b9c 100644 --- a/tslang/include/TypeScript/Defines.h +++ b/tslang/include/TypeScript/Defines.h @@ -414,6 +414,9 @@ // `@value class C` / `@value interface R`: a type whose values are copied, not shared - no identity, // no vtable, no base (docs/superpowers/specs/2026-10-24-value-types-design.md) #define VALUE_DECORATOR "value" +// `@atomic class`: objects several threads may use at once (ClassInfo::isAtomic). On a variable, +// an atomic load and store (MLIRGenVariables). +#define ATOMIC_DECORATOR "atomic" // `@sharedElements class Map`: a container that keeps what it is given and hands out // borrows of it. Under -mm=own each of its type arguments has to be a value, a string (copied) or // a Shared handle (counted): an object, whose block has one owner, goes through Shared. @@ -505,6 +508,13 @@ // ABI break. See docs/reference-counting-evaluation.md sections 9.5 and 9.24. #define HEAP_BLOCK_IMMORTAL -1 +// The top bit of a counted block's header word says the block is shared: other threads may hold +// references to it, so its count changes are atomic (OwnershipRoutineLogic::emitIncRef and +// emitDecRef). The rest of the word is the count, which never grows into the bit. A block's bit +// is set once, by the thread that holds it before any other can reach it, and never cleared. +// HEAP_BLOCK_IMMORTAL has the bit set too, so an immortal test always comes first. It is the +// word's top bit, so the value depends on the word size: 1 << (bits - 1). + // Runtime type descriptor. // // The ANY_TYPE slot of an "any" box, and the UNION_TAG_INDEX slot of a tagged union, both @@ -534,12 +544,16 @@ // because a tagged union carries its payload inline: copying or dropping one has to retain // or release a value whose type is only known at run time, and the tag is what knows it. #define TYPE_DESCR_RETAIN 3 +// Address of the type's share routine (`Shared.share`, OwnershipRoutineLogic::getOrCreateShareRoutine), +// on the same terms again: it sets the shared bit of every block a value whose type is only known +// at run time reaches. Null under every model but `rc`. +#define TYPE_DESCR_SHARE 4 // The block header, last so that it sits immediately in front of the name bytes. A tag is a // `typeof` result, and `typeof x` can be assigned to a `string` and released like any other // string - so a tag has to look like a payload with an immortal block header, exactly as a // string literal does, on top of being a name preceded by a descriptor. Both reads work off // the same pointer: `tag - sizeof(header)` is the marker, `tag - sizeof(record)` the record. -#define TYPE_DESCR_BLOCK_HEADER 4 +#define TYPE_DESCR_BLOCK_HEADER 5 // Coarse category of the described type. These correspond one-to-one with the names // TypeOfOpHelper::typeOfAsString reports, and are derived from that name so the two cannot diff --git a/tslang/include/TypeScript/LowerToLLVM/LLVMCodeHelper.h b/tslang/include/TypeScript/LowerToLLVM/LLVMCodeHelper.h index 507fb1dd8..a02cf981e 100644 --- a/tslang/include/TypeScript/LowerToLLVM/LLVMCodeHelper.h +++ b/tslang/include/TypeScript/LowerToLLVM/LLVMCodeHelper.h @@ -354,7 +354,7 @@ class LLVMCodeHelper : public LLVMCodeHelperBase // capture box); it is not part of the name the record carries. mlir::Value getOrCreateTypeDescriptorName(mlir::Type type, std::string name, int kind, StringRef releaseRoutineName, StringRef retainRoutineName, - StringRef variant = "", uint32_t layout = 0) + StringRef shareRoutineName, StringRef variant = "", uint32_t layout = 0) { auto loc = op->getLoc(); auto parentModule = op->getParentOfType(); @@ -405,6 +405,11 @@ class LLVMCodeHelper : public LLVMCodeHelperBase ? (mlir::Value)rewriter.create(loc, th.getPtrType()) : (mlir::Value)rewriter.create(loc, th.getPtrType(), retainRoutineName); setStructValue(loc, recordValue, retainValue, TYPE_DESCR_RETAIN); + mlir::Value shareValue = + shareRoutineName.empty() + ? (mlir::Value)rewriter.create(loc, th.getPtrType()) + : (mlir::Value)rewriter.create(loc, th.getPtrType(), shareRoutineName); + setStructValue(loc, recordValue, shareValue, TYPE_DESCR_SHARE); // a tag doubles as a string payload, so what precedes the name has to read as an // immortal block header - see TYPE_DESCR_BLOCK_HEADER setStructValue(loc, recordValue, diff --git a/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h b/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h index 680cd8b81..c44ce37ae 100644 --- a/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h +++ b/tslang/include/TypeScript/LowerToLLVM/OwnershipRoutineLogic.h @@ -524,6 +524,148 @@ class OwnershipRoutineLogic }); } + // === Sharing between threads === + // + // `Shared.share(v)` (ts.MarkShared) sets the shared bit (Defines.h, beside HEAP_BLOCK_IMMORTAL) + // in the header of every counted block `v` reaches, after which emitIncRef and emitDecRef + // change those counts atomically. "Reaches" is what the release routine of the same type would + // give back - fields, elements, an array's data block, a closure's capture box and its cells, + // and through a descriptor (TYPE_DESCR_SHARE) whatever an `any`, an interface or a union holds - + // so that no block another thread can reach through `v` is counted plainly. A block that is + // already shared stops the walk below it: what it holds was marked when it was, and that is + // also what ends the walk on a cycle. An immortal block is left alone, and has to be: a + // literal's header may be in read-only memory. + // + // Only `rc` keeps counts on every block; under the other models the op is erased. + + // Symbol name of the share routine for `type`, generating it if needed; empty when the type + // owns no heap memory. Like release, it takes the storage holding a value. + std::string getOrCreateShareRoutine(mlir::Type type) + { + if (!compileOptions.isRefCounted() || !ownsHeapMemory(type)) + { + return {}; + } + + std::stringstream nameStream; + nameStream << "tsshr_" << (size_t)hash_value(type); + auto name = nameStream.str(); + + auto parentModule = op->getParentOfType(); + if (parentModule.lookupSymbol(name)) + { + return name; + } + + TypeHelper th(rewriter); + auto loc = op->getLoc(); + + OpBuilder::InsertionGuard insertGuard(rewriter); + rewriter.setInsertionPointToStart(parentModule.getBody()); + + auto funcOp = rewriter.create( + loc, name, th.getFunctionType(th.getVoidType(), {th.getPtrType()}), LLVM::Linkage::Internal); + + // the symbol exists before the body, for a recursive type, as the release routine's does + auto *entryBlock = funcOp.addEntryBlock(rewriter); + rewriter.setInsertionPointToStart(entryBlock); + + buildShareBody(type, entryBlock->getArgument(0)); + + rewriter.create(loc, ValueRange{}); + + dropDebugLocations(funcOp); + return name; + } + + // The share routine of a closure's capture box: the box, each captured variable's cell and what + // the cell holds, and what each field captured by value holds. Keyed by the capture's + // `ref>`, as the box's release routine is. + std::string getOrCreateCaptureBoxShareRoutine(mlir_ts::RefType captureRefType) + { + if (!compileOptions.isRefCounted()) + { + return {}; + } + + std::stringstream nameStream; + nameStream << "tsshrbox_" << (size_t)hash_value(captureRefType); + auto name = nameStream.str(); + + auto parentModule = op->getParentOfType(); + if (parentModule.lookupSymbol(name)) + { + return name; + } + + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + auto loc = op->getLoc(); + auto ptrTy = th.getPtrType(); + + OpBuilder::InsertionGuard insertGuard(rewriter); + rewriter.setInsertionPointToStart(parentModule.getBody()); + + auto funcOp = rewriter.create(loc, name, th.getFunctionType(th.getVoidType(), {ptrTy}), + LLVM::Linkage::Internal); + + auto *entryBlock = funcOp.addEntryBlock(rewriter); + rewriter.setInsertionPointToStart(entryBlock); + + auto tupleType = captureRefType.getElementType(); + auto captureTupleType = dyn_cast(tupleType); + auto boxValue = rewriter.create(loc, ptrTy, entryBlock->getArgument(0)); + emitMarkBlockShared(boxValue, [&]() { + auto llvmTupleType = tch.convertType(tupleType); + for (auto [index, fieldType] : llvm::enumerate(getFieldTypes(tupleType))) + { + // a @value constructor's `this` is the caller's slot, not a cell (releaseCapturedFields) + if (captureTupleType && + MLIRTypeHelper::isValueConstructorThisCapture(captureTupleType.getFields()[index].id, fieldType)) + { + continue; + } + + auto refFieldType = dyn_cast(fieldType); + if (!refFieldType && !ownsHeapMemory(fieldType)) + { + continue; + } + + auto fieldPtr = rewriter.create(loc, ptrTy, llvmTupleType, boxValue, + ArrayRef{0, (int32_t)index}); + if (refFieldType) + { + // the field holds the cell's address: the cell is a counted block of its own + auto cellValue = rewriter.create(loc, ptrTy, fieldPtr); + emitMarkBlockShared(cellValue, [&]() { shareSlot(refFieldType.getElementType(), cellValue); }); + } + else + { + shareSlot(fieldType, fieldPtr); + } + } + }); + + rewriter.create(loc, ValueRange{}); + + dropDebugLocations(funcOp); + return name; + } + + // ts.MarkShared: the share routine of `type` on `value`. + void emitShareValue(mlir::Type type, mlir::Value value) + { + auto wrapperName = getOrCreateValueWrapper(type, getOrCreateShareRoutine(type), "tsshrv_"); + if (wrapperName.empty()) + { + return; + } + + rewriter.create(op->getLoc(), TypeRange{}, FlatSymbolRefAttr::get(rewriter.getContext(), wrapperName), + ValueRange{value}); + } + private: // Field types of a record-shaped type, empty for anything else. llvm::SmallVector getFieldTypes(mlir::Type type) @@ -644,20 +786,37 @@ class OwnershipRoutineLogic // A block marked HEAP_BLOCK_IMMORTAL is neither decremented nor ever the last. That is what // lets a string literal, or a `typeof` result pointing into a descriptor, be released like // any other string without writing to read-only memory or freeing a static block. + // + // A block with the shared bit set (other threads may hold references to it) + // is decremented with an atomic subtract with release ordering, and the last reference is the + // one that saw a count of 1 before it: two threads dropping the last two references at once + // must not both see 1 left, nor both see 0. The thread that frees then takes an acquire fence, + // so the writes other threads made to the object before giving up their references happen + // before its destruction. Any other block - the common case, one thread's own - is a plain + // load, subtract and store, as it always was. Under --atomic-rc every block takes the atomic + // path, shared bit or not. + // + // The first read is a plain load (emitCountLoad says why that is enough). An immortal header + // is never written again, and a block's shared bit is only ever set, by the one thread that + // holds the block before any other can reach it, so neither test can be made stale by another + // thread. Marking a block immortal (emitMarkImmortal) stays a plain + // write: it is done to globals and keepers before any other thread can reach the block. mlir::Value emitDecRef(mlir::Value payloadPtr) { TypeHelper th(rewriter); auto loc = op->getLoc(); auto parentModule = op->getParentOfType(); - const char *helperName = "__tslang_dec_ref"; + const bool atomicAlways = compileOptions.atomicRefCounts(); + const char *helperName = atomicAlways ? "__tslang_dec_ref_atomic" : "__tslang_dec_ref"; if (!parentModule.lookupSymbol(helperName)) { OpBuilder::InsertionGuard insertGuard(rewriter); rewriter.setInsertionPointToStart(parentModule.getBody()); + auto boolType = th.getLLVMBoolType(); auto helper = rewriter.create( - loc, helperName, th.getFunctionType(th.getLLVMBoolType(), {th.getPtrType()}), LLVM::Linkage::Internal); + loc, helperName, th.getFunctionType(boolType, {th.getPtrType()}), LLVM::Linkage::Internal); auto *entryBlock = helper.addEntryBlock(rewriter); rewriter.setInsertionPointToStart(entryBlock); @@ -667,29 +826,51 @@ class OwnershipRoutineLogic auto llvmIndexType = tch.convertType(th.getIndexType()); auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, entryBlock->getArgument(0), llvmIndexType); - auto count = rewriter.create(loc, llvmIndexType, blockPtr); - auto immortal = rewriter.create( - loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, HEAP_BLOCK_IMMORTAL)); - auto isMortal = rewriter.create(loc, LLVM::ICmpPredicate::ne, count, immortal); + auto count = emitCountLoad(loc, llvmIndexType, blockPtr); - auto *decBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); - auto *immortalBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *sharedBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *plainBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *lastBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *falseBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); rewriter.setInsertionPointToEnd(entryBlock); - rewriter.create(loc, isMortal, decBlock, immortalBlock); + emitBranchOnHeaderWord(loc, llvmIndexType, count, atomicAlways, plainBlock, sharedBlock, falseBlock); - rewriter.setInsertionPointToStart(decBlock); - auto one = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); - auto newCount = rewriter.create(loc, llvmIndexType, count, one); - rewriter.create(loc, newCount, blockPtr); - auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); - auto wasLast = rewriter.create(loc, LLVM::ICmpPredicate::eq, newCount, zero); - rewriter.create(loc, ValueRange{wasLast}); + auto one = [&]() { + return rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); + }; + + // shared: the count without the bit was 1 before this thread's subtract + rewriter.setInsertionPointToStart(sharedBlock); + auto oldWord = rewriter.create(loc, LLVM::AtomicBinOp::sub, blockPtr, one(), + LLVM::AtomicOrdering::release); + auto oldCount = emitWithoutSharedBit(loc, llvmIndexType, oldWord); + auto wasLastShared = rewriter.create(loc, LLVM::ICmpPredicate::eq, oldCount, one()); + rewriter.create(loc, wasLastShared, lastBlock, falseBlock); - rewriter.setInsertionPointToStart(immortalBlock); + rewriter.setInsertionPointToStart(lastBlock); + rewriter.create(loc, LLVM::AtomicOrdering::acquire); rewriter.create( - loc, ValueRange{rewriter.create(loc, th.getLLVMBoolType(), - rewriter.getIntegerAttr(th.getLLVMBoolType(), 0))}); + loc, ValueRange{rewriter.create(loc, boolType, rewriter.getIntegerAttr(boolType, 1))}); + + // this thread's own: the bit is clear, so the word is the count + rewriter.setInsertionPointToStart(plainBlock); + if (atomicAlways) + { + rewriter.create(loc, ValueRange{}, falseBlock); + } + else + { + auto newCount = rewriter.create(loc, llvmIndexType, count, one()); + rewriter.create(loc, newCount, blockPtr); + auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); + auto wasLast = rewriter.create(loc, LLVM::ICmpPredicate::eq, newCount, zero); + rewriter.create(loc, ValueRange{wasLast}); + } + + rewriter.setInsertionPointToStart(falseBlock); + rewriter.create( + loc, ValueRange{rewriter.create(loc, boolType, rewriter.getIntegerAttr(boolType, 0))}); dropDebugLocations(helper); } @@ -705,13 +886,19 @@ class OwnershipRoutineLogic // Skipping an immortal block is not an optimisation: incrementing HEAP_BLOCK_IMMORTAL // would turn all-ones into zero, and the next release would read that as "last reference" // and free a string literal. + // + // A shared block (or any, under --atomic-rc) takes an atomic add. Relaxed ordering is enough: + // whoever takes a reference already holds one, so the block cannot be freed under it, and + // nothing else is published by taking a reference. The count never reaches the shared bit, + // so the add leaves the bit alone. void emitIncRef(mlir::Value payloadPtr) { TypeHelper th(rewriter); auto loc = op->getLoc(); auto parentModule = op->getParentOfType(); - const char *helperName = "__tslang_inc_ref"; + const bool atomicAlways = compileOptions.atomicRefCounts(); + const char *helperName = atomicAlways ? "__tslang_inc_ref_atomic" : "__tslang_inc_ref"; if (!parentModule.lookupSymbol(helperName)) { OpBuilder::InsertionGuard insertGuard(rewriter); @@ -733,7 +920,8 @@ class OwnershipRoutineLogic auto isNotNull = rewriter.create(loc, LLVM::ICmpPredicate::ne, payload, nullPtr); auto *loadBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); - auto *incBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *sharedBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); + auto *plainBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); auto *returnBlock = rewriter.createBlock(&helper.getBody(), helper.getBody().end()); rewriter.setInsertionPointToEnd(entryBlock); @@ -741,15 +929,24 @@ class OwnershipRoutineLogic rewriter.setInsertionPointToStart(loadBlock); auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, payload, llvmIndexType); - auto count = rewriter.create(loc, llvmIndexType, blockPtr); - auto immortal = rewriter.create( - loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, HEAP_BLOCK_IMMORTAL)); - auto isMortal = rewriter.create(loc, LLVM::ICmpPredicate::ne, count, immortal); - rewriter.create(loc, isMortal, incBlock, returnBlock); - - rewriter.setInsertionPointToStart(incBlock); - auto one = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); - rewriter.create(loc, rewriter.create(loc, llvmIndexType, count, one), blockPtr); + auto count = emitCountLoad(loc, llvmIndexType, blockPtr); + emitBranchOnHeaderWord(loc, llvmIndexType, count, atomicAlways, plainBlock, sharedBlock, returnBlock); + + auto one = [&]() { + return rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 1)); + }; + + rewriter.setInsertionPointToStart(sharedBlock); + rewriter.create(loc, LLVM::AtomicBinOp::add, blockPtr, one(), + LLVM::AtomicOrdering::monotonic); + rewriter.create(loc, ValueRange{}, returnBlock); + + rewriter.setInsertionPointToStart(plainBlock); + if (!atomicAlways) + { + rewriter.create(loc, rewriter.create(loc, llvmIndexType, count, one()), + blockPtr); + } rewriter.create(loc, ValueRange{}, returnBlock); rewriter.setInsertionPointToStart(returnBlock); @@ -762,6 +959,69 @@ class OwnershipRoutineLogic ValueRange{payloadPtr}); } + // The read of a block's header word by the count helpers. A plain load, though a shared block's + // word may be changing under other threads' atomic adds and subtracts: all the helpers decide + // from it is which kind of block this is, and that cannot change under them. A word only this + // thread holds is written by this thread alone; a shared block's top bit is never cleared, and + // an immortal word never written, so either reads negative whatever the count does. The count + // of a shared block is never taken from this read - its atomic subtract returns its own. An + // atomic load would say the same, but it keeps LLVM from cancelling a retain against the + // release that follows it on a block this thread holds, which is most of what reference + // counting costs in a hot loop (the ray tracer: 4.5% over the counts before blocks could be + // shared, 7% with an atomic load here). + mlir::Value emitCountLoad(mlir::Location loc, mlir::Type llvmIndexType, mlir::Value blockPtr) + { + return rewriter.create(loc, llvmIndexType, blockPtr); + } + + // The shared bit of a block's header word: the top bit (Defines.h, beside HEAP_BLOCK_IMMORTAL). + // A count never grows into it, and HEAP_BLOCK_IMMORTAL (all ones) has it set too - which is why + // every helper tests for immortal first. + mlir::Value emitSharedBitConstant(mlir::Location loc, mlir::Type llvmIndexType) + { + auto bits = compileOptions.sizeBits(); + return rewriter.create( + loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, llvm::APInt::getOneBitSet(bits, bits - 1))); + } + + // Ends the current block with the three-way branch of the count helpers on a header word: + // `plainBlock` for a block only this thread holds, `sharedBlock` for a shared one, and + // `immortalBlock` for an immortal one. The shared bit is the word's top bit, which + // HEAP_BLOCK_IMMORTAL has set as well, so one signed test - the word is not negative - is all + // the common case pays, as it paid one test for immortal before blocks could be shared. Under + // --atomic-rc every mortal block is treated as shared. + void emitBranchOnHeaderWord(mlir::Location loc, mlir::Type llvmIndexType, mlir::Value word, bool atomicAlways, + mlir::Block *plainBlock, mlir::Block *sharedBlock, mlir::Block *immortalBlock) + { + auto immortal = rewriter.create( + loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, HEAP_BLOCK_IMMORTAL)); + auto isMortal = rewriter.create(loc, LLVM::ICmpPredicate::ne, word, immortal); + if (atomicAlways) + { + rewriter.create(loc, isMortal, sharedBlock, immortalBlock); + return; + } + + auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); + auto isPlain = rewriter.create(loc, LLVM::ICmpPredicate::sge, word, zero); + + auto *currentBlock = rewriter.getInsertionBlock(); + auto *topBitBlock = rewriter.createBlock(currentBlock->getParent(), std::next(currentBlock->getIterator())); + rewriter.create(loc, isMortal, sharedBlock, immortalBlock); + + rewriter.setInsertionPointToEnd(currentBlock); + rewriter.create(loc, isPlain, plainBlock, topBitBlock); + } + + // The count a header word holds, without the shared bit. + mlir::Value emitWithoutSharedBit(mlir::Location loc, mlir::Type llvmIndexType, mlir::Value word) + { + auto bits = compileOptions.sizeBits(); + auto mask = rewriter.create( + loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, llvm::APInt::getLowBitsSet(bits, bits - 1))); + return rewriter.create(loc, llvmIndexType, word, mask); + } + // Is this block mortal - that is, not HEAP_BLOCK_IMMORTAL? Inline rather than a helper: it // is one load and one compare, and it is the whole of what a release costs under own. mlir::Value emitIsMortal(mlir::Value payloadPtr) @@ -1600,6 +1860,281 @@ class OwnershipRoutineLogic rewriter.setInsertionPointToStart(continuationBlock); } + + // Sets the shared bit of one block and then runs `thenBody` - the walk into what the block + // holds - only when this call is what set it: not for null, not for an immortal block, not for + // one already shared. The bit is the word's top bit, which an immortal word has too, so a word + // that is not negative is exactly a block to mark. An atomic `or`: the count is left as it is. + void emitMarkBlockShared(mlir::Value payloadPtr, llvm::function_ref thenBody) + { + emitIfNonNull(payloadPtr, [&]() { + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + LLVMCodeHelperBase ch(op, rewriter, typeConverter, compileOptions); + auto loc = op->getLoc(); + + auto llvmIndexType = tch.convertType(th.getIndexType()); + auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, payloadPtr, llvmIndexType); + auto word = emitCountLoad(loc, llvmIndexType, blockPtr); + auto zero = rewriter.create(loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, 0)); + auto isUnmarked = rewriter.create(loc, LLVM::ICmpPredicate::sge, word, zero); + + auto *currentBlock = rewriter.getInsertionBlock(); + auto *continuationBlock = rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint()); + auto *markBlock = rewriter.createBlock(continuationBlock); + + rewriter.setInsertionPointToEnd(markBlock); + rewriter.create(loc, LLVM::AtomicBinOp::_or, blockPtr, + emitSharedBitConstant(loc, llvmIndexType), + LLVM::AtomicOrdering::monotonic); + thenBody(); + rewriter.create(loc, ValueRange{}, continuationBlock); + + rewriter.setInsertionPointToEnd(currentBlock); + rewriter.create(loc, isUnmarked, markBlock, continuationBlock); + rewriter.setInsertionPointToStart(continuationBlock); + }); + } + + // Calls the share routine of `type` on `slotPtr`, if it has one. + void shareSlot(mlir::Type type, mlir::Value slotPtr) + { + auto routineName = getOrCreateShareRoutine(type); + if (routineName.empty()) + { + return; + } + + rewriter.create(op->getLoc(), TypeRange{}, + FlatSymbolRefAttr::get(rewriter.getContext(), routineName), ValueRange{slotPtr}); + } + + // Calls the routine a descriptor holds in `descriptorSlot` (TYPE_DESCR_*) on `valueSlotPtr`, if + // it holds one. `tagValue` must not be null. + void callViaDescriptor(mlir::Value tagValue, mlir::Value valueSlotPtr, int32_t descriptorSlot) + { + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + TypeDescriptorLogic tdl(rewriter, tch, op->getLoc()); + + auto loc = op->getLoc(); + + auto recordPtr = tdl.getRecordPtrFromTag(tagValue); + auto routinePtrSlot = + rewriter.create(loc, th.getPtrType(), + TypeDescriptorLogic::getRecordType(rewriter, tch.convertType(th.getIndexType())), + recordPtr, ArrayRef{0, descriptorSlot}); + auto routine = rewriter.create(loc, th.getPtrType(), routinePtrSlot); + + emitIfNonNull(routine, [&]() { + mlir::SmallVector ops{routine, valueSlotPtr}; + auto callOp = rewriter.create(loc, TypeRange{}, ops); + callOp.getProperties().setOperandSegmentSizes({static_cast(ops.size()), 0}); + callOp.setOpBundleSizes({}); + }); + } + + // The fields of a record-shaped value, `basePtr` addressing the record. Every field the release + // routine would visit, and the ones it leaves to an owner elsewhere too: marking more than + // another thread can reach costs atomic counts, marking less costs a race. + void shareFields(mlir::Type recordType, mlir::Value basePtr) + { + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + + auto loc = op->getLoc(); + auto llvmRecordType = tch.convertType(recordType); + + for (auto [index, fieldType] : llvm::enumerate(getFieldTypes(recordType))) + { + auto routineName = std::string(); + if (auto refFieldType = dyn_cast(fieldType)) + { + // a `ref` to a tuple is a capture box (releaseFields); any other `ref` owns nothing + if (isa(refFieldType.getElementType())) + { + routineName = getOrCreateCaptureBoxShareRoutine(refFieldType); + } + } + else + { + routineName = getOrCreateShareRoutine(fieldType); + } + + if (routineName.empty()) + { + continue; + } + + auto fieldPtr = rewriter.create(loc, th.getPtrType(), llvmRecordType, basePtr, + ArrayRef{0, (int32_t)index}); + rewriter.create(loc, TypeRange{}, FlatSymbolRefAttr::get(rewriter.getContext(), routineName), + ValueRange{fieldPtr}); + } + } + + // The share routine's body, case for case with buildBody's. + void buildShareBody(mlir::Type typeIn, mlir::Value slotPtr) + { + auto type = MLIRTypeHelper::stripValueType(typeIn); + + TypeHelper th(rewriter); + TypeConverterHelper tch(typeConverter); + + auto loc = op->getLoc(); + auto ptrTy = th.getPtrType(); + + if (isa(type)) + { + auto strValue = rewriter.create(loc, ptrTy, slotPtr); + emitMarkBlockShared(strValue, [&]() {}); + return; + } + + // the header, then its data block, then the elements [0, length) - each only when the one + // before it was not shared already + if (auto arrayType = dyn_cast(type)) + { + ArrayLayout layout(op, rewriter, typeConverter, compileOptions); + auto header = layout.headerForRead(arrayType, slotPtr); + emitMarkBlockShared(header, [&]() { + auto dataValue = rewriter.create(loc, ptrTy, layout.dataAddress(arrayType, header)); + emitMarkBlockShared(dataValue, [&]() { + auto elementRoutine = getOrCreateShareRoutine(arrayType.getElementType()); + if (elementRoutine.empty()) + { + return; + } + + auto llvmIndexType = tch.convertType(th.getIndexType()); + auto sizeValue = rewriter.create(loc, llvmIndexType, layout.lengthAddress(arrayType, header)); + emitCountedLoop(sizeValue, [&](mlir::Value index) { + auto llvmElementType = tch.convertType(arrayType.getElementType()); + auto elementPtr = + rewriter.create(loc, ptrTy, llvmElementType, dataValue, ValueRange{index}); + rewriter.create(loc, TypeRange{}, + FlatSymbolRefAttr::get(rewriter.getContext(), elementRoutine), + ValueRange{elementPtr}); + }); + }); + }); + return; + } + + if (isa(type) || isa(type)) + { + auto storageType = isa(type) ? cast(type).getStorageType() + : cast(type).getStorageType(); + + auto instanceValue = rewriter.create(loc, ptrTy, slotPtr); + emitMarkBlockShared(instanceValue, [&]() { shareFields(storageType, instanceValue); }); + return; + } + + if (auto sharedType = dyn_cast(type)) + { + auto handle = rewriter.create(loc, ptrTy, slotPtr); + emitMarkBlockShared(handle, [&]() { shareSlot(sharedType.getElementType(), handle); }); + return; + } + + if (isa(type)) + { + auto boxValue = rewriter.create(loc, ptrTy, slotPtr); + emitMarkBlockShared(boxValue, [&]() { + auto anyStructType = LLVM::LLVMStructType::getLiteral( + rewriter.getContext(), {tch.convertType(th.getIndexType()), ptrTy, th.getI8Type()}, false); + + auto tagSlot = rewriter.create(loc, ptrTy, anyStructType, boxValue, + ArrayRef{0, ANY_TYPE}); + auto tagValue = rewriter.create(loc, ptrTy, tagSlot); + auto dataSlot = rewriter.create(loc, ptrTy, anyStructType, boxValue, + ArrayRef{0, ANY_DATA}); + emitIfNonNull(tagValue, [&]() { callViaDescriptor(tagValue, dataSlot, TYPE_DESCR_SHARE); }); + }); + return; + } + + // { vtable, this }: the tag of what `this` points at is the vtable's first slot + if (isa(type)) + { + auto llvmType = tch.convertType(type); + auto vtableSlot = rewriter.create(loc, ptrTy, llvmType, slotPtr, + ArrayRef{0, DATA_VALUE_INDEX}); + auto vtableValue = rewriter.create(loc, ptrTy, vtableSlot); + auto thisSlot = rewriter.create(loc, ptrTy, llvmType, slotPtr, + ArrayRef{0, THIS_VALUE_INDEX}); + emitIfNonNull(vtableValue, [&]() { + auto tagSlot = rewriter.create(loc, ptrTy, ptrTy, vtableValue, + ArrayRef{INTERFACE_VTABLE_TYPE_SLOT}); + auto tagValue = rewriter.create(loc, ptrTy, tagSlot); + emitIfNonNull(tagValue, [&]() { callViaDescriptor(tagValue, thisSlot, TYPE_DESCR_SHARE); }); + }); + return; + } + + // { func, this, type }: the tag beside `this` says what `this` is - a capture box, or the + // object a method is bound to + if (isa(type) || isa(type)) + { + auto llvmType = tch.convertType(type); + auto tagSlot = rewriter.create(loc, ptrTy, llvmType, slotPtr, + ArrayRef{0, CLOSURE_TYPE_INDEX}); + auto tagValue = rewriter.create(loc, ptrTy, tagSlot); + auto thisSlot = rewriter.create(loc, ptrTy, llvmType, slotPtr, + ArrayRef{0, THIS_VALUE_INDEX}); + emitIfNonNull(tagValue, [&]() { callViaDescriptor(tagValue, thisSlot, TYPE_DESCR_SHARE); }); + return; + } + + if (auto unionType = dyn_cast(type)) + { + MLIRTypeHelper mth(rewriter.getContext(), compileOptions); + mlir::Type baseType; + if (mth.isUnionTypeNeedsTag(loc, unionType, baseType)) + { + auto llvmUnionType = tch.convertType(unionType); + auto tagSlot = rewriter.create(loc, ptrTy, llvmUnionType, slotPtr, + ArrayRef{0, UNION_TAG_INDEX}); + auto tagValue = rewriter.create(loc, ptrTy, tagSlot); + auto valueSlot = rewriter.create(loc, ptrTy, llvmUnionType, slotPtr, + ArrayRef{0, UNION_VALUE_INDEX}); + emitIfNonNull(tagValue, [&]() { callViaDescriptor(tagValue, valueSlot, TYPE_DESCR_SHARE); }); + } + else + { + shareSlot(baseType, slotPtr); + } + + return; + } + + if (auto optionalType = dyn_cast(type)) + { + auto llvmOptionalType = tch.convertType(optionalType); + auto hasValueSlot = rewriter.create(loc, ptrTy, llvmOptionalType, slotPtr, + ArrayRef{0, OPTIONAL_HASVALUE_INDEX}); + auto hasValue = rewriter.create(loc, th.getLLVMBoolType(), hasValueSlot); + + auto *currentBlock = rewriter.getInsertionBlock(); + auto *continuationBlock = rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint()); + auto *thenBlock = rewriter.createBlock(continuationBlock); + + rewriter.setInsertionPointToEnd(thenBlock); + auto valueSlot = rewriter.create(loc, ptrTy, llvmOptionalType, slotPtr, + ArrayRef{0, OPTIONAL_VALUE_INDEX}); + shareSlot(optionalType.getElementType(), valueSlot); + rewriter.create(loc, ValueRange{}, continuationBlock); + + rewriter.setInsertionPointToEnd(currentBlock); + rewriter.create(loc, hasValue, thenBlock, continuationBlock); + rewriter.setInsertionPointToStart(continuationBlock); + return; + } + + // record-shaped and inline: the value's storage is its holder's, only its fields reach blocks + shareFields(type, slotPtr); + } }; } // namespace typescript diff --git a/tslang/include/TypeScript/LowerToLLVM/TypeDescriptorLogic.h b/tslang/include/TypeScript/LowerToLLVM/TypeDescriptorLogic.h index f0273fcc2..17c380923 100644 --- a/tslang/include/TypeScript/LowerToLLVM/TypeDescriptorLogic.h +++ b/tslang/include/TypeScript/LowerToLLVM/TypeDescriptorLogic.h @@ -37,7 +37,7 @@ class TypeDescriptorLogic { auto i32Ty = builder.getI32Type(); auto ptrTy = LLVM::LLVMPointerType::get(builder.getContext()); - return LLVM::LLVMStructType::getLiteral(builder.getContext(), {i32Ty, i32Ty, ptrTy, ptrTy, llvmIndexType}, false); + return LLVM::LLVMStructType::getLiteral(builder.getContext(), {i32Ty, i32Ty, ptrTy, ptrTy, ptrTy, llvmIndexType}, false); } LLVM::LLVMStructType getRecordType() diff --git a/tslang/include/TypeScript/MLIRLogic/MLIRGenStore.h b/tslang/include/TypeScript/MLIRLogic/MLIRGenStore.h index 9d45cd187..10303a971 100644 --- a/tslang/include/TypeScript/MLIRLogic/MLIRGenStore.h +++ b/tslang/include/TypeScript/MLIRLogic/MLIRGenStore.h @@ -839,6 +839,31 @@ struct ClassInfo // else, a slot left to a declared C++ method null), may have a constructor and field // initializers, and `.new`; still no RTTI. tslang allocates its objects. bool isCppVTableImplementation = false; + // `@atomic`: its objects may be used by several threads at once. Under -mm=rc a new object is + // marked shared, with everything it holds, once its constructor has run, and a value stored + // into one of its fields is marked before the store (ts.MarkShared), so every block reachable + // from it counts atomically. A class extending one is @atomic too (isAtomicClass). Its fields' + // own consistency is still the program's to keep. + bool isAtomic = false; + + // this class or a base class is @atomic + bool isAtomicClass() const + { + if (isAtomic) + { + return true; + } + + for (auto &baseClass : baseClasses) + { + if (baseClass && baseClass->isAtomicClass()) + { + return true; + } + } + + return false; + } // two @cpp methods in one slot, or an override's @slot not its base's: found by // getVirtualTable, which can't report it, and reported by its caller std::string cppSlotConflict; diff --git a/tslang/include/TypeScript/TypeScriptOps.td b/tslang/include/TypeScript/TypeScriptOps.td index ac7a6fd36..978049a2b 100644 --- a/tslang/include/TypeScript/TypeScriptOps.td +++ b/tslang/include/TypeScript/TypeScriptOps.td @@ -571,6 +571,25 @@ def TypeScript_ReleaseOp : TypeScript_Op<"Release"> { let arguments = (ins AnyType:$reference); } +def TypeScript_MarkSharedOp : TypeScript_Op<"MarkShared"> { + let summary = "mark a value, and every block it reaches, shared between threads"; + let description = [{ + `Shared.share(v)`: sets the shared bit (Defines.h, beside HEAP_BLOCK_IMMORTAL) in the header + of every counted block $value reaches - the blocks its release would give back - so that + their count changes are atomic from here on and other threads may take and drop references + to them. A block already shared stops the walk, which is also what ends it on a cycle, and an + immortal one is left alone. Takes no reference and keeps none. + + What is stored into a shared object afterwards is not marked by this: it has to be shared + first (or be stored into a field of an `@atomic` class, which marks what it is given - but + not an element pushed into an array such a field holds). + + Erased under a memory model that keeps no counts. + }]; + + let arguments = (ins AnyType:$value); +} + def TypeScript_RetainSlotOp : TypeScript_Op<"RetainSlot"> { let summary = "take one reference to the value held in a slot"; let description = [{ diff --git a/tslang/lib/TypeScript/LowerToAffineLoops.cpp b/tslang/lib/TypeScript/LowerToAffineLoops.cpp index ee13f9cc6..1254c9f10 100644 --- a/tslang/lib/TypeScript/LowerToAffineLoops.cpp +++ b/tslang/lib/TypeScript/LowerToAffineLoops.cpp @@ -3107,7 +3107,7 @@ void AddTsAffineLegalOps(ConversionTarget &target) mlir_ts::AddressOfOp, mlir_ts::ArithmeticBinaryOp, mlir_ts::ArithmeticUnaryOp, mlir_ts::AssertOp, mlir_ts::CastOp, mlir_ts::ConstantOp, mlir_ts::ElementRefOp, mlir_ts::PointerOffsetRefOp, mlir_ts::FuncOp, mlir_ts::GlobalOp, mlir_ts::GlobalResultOp, mlir_ts::DefaultOp, mlir_ts::HasValueOp, mlir_ts::ValueOp, mlir_ts::ValueOrDefaultOp, mlir_ts::NullOp, mlir_ts::ParseFloatOp, mlir_ts::ParseIntOp, mlir_ts::IsNaNOp, - mlir_ts::PrintOp, mlir_ts::ConvertFOp, mlir_ts::SizeOfOp, mlir_ts::TypeDescriptorOp, mlir_ts::RetainOp, mlir_ts::ReleaseOp, mlir_ts::RetainSlotOp, mlir_ts::ReleaseSlotOp, mlir_ts::RetainCellOp, mlir_ts::ReleaseCellOp, mlir_ts::StoreOp, mlir_ts::SwapOp, mlir_ts::SymbolRefOp, mlir_ts::LengthOfOp, mlir_ts::SetLengthOfOp, + mlir_ts::PrintOp, mlir_ts::ConvertFOp, mlir_ts::SizeOfOp, mlir_ts::TypeDescriptorOp, mlir_ts::RetainOp, mlir_ts::ReleaseOp, mlir_ts::MarkSharedOp, mlir_ts::RetainSlotOp, mlir_ts::ReleaseSlotOp, mlir_ts::RetainCellOp, mlir_ts::ReleaseCellOp, mlir_ts::StoreOp, mlir_ts::SwapOp, mlir_ts::SymbolRefOp, mlir_ts::LengthOfOp, mlir_ts::SetLengthOfOp, mlir_ts::StringLengthOp, mlir_ts::StringResizeOp, mlir_ts::StringConcatOp, mlir_ts::StringCopyOp, mlir_ts::SharedNewOp, mlir_ts::SharedValueRefOp, mlir_ts::SharedCountOp, mlir_ts::StringCompareOp, mlir_ts::AnyCompareOp, mlir_ts::LoadOp, mlir_ts::LoadSaveOp, mlir_ts::NewOp, mlir_ts::CreateTupleOp, mlir_ts::DeconstructTupleOp, mlir_ts::CreateArrayOp, mlir_ts::NewEmptyArrayOp, mlir_ts::NewArrayOp, mlir_ts::DeleteOp, mlir_ts::PropertyRefOp, mlir_ts::InsertPropertyOp, diff --git a/tslang/lib/TypeScript/LowerToLLVM.cpp b/tslang/lib/TypeScript/LowerToLLVM.cpp index a1a4b81c8..3ee622a6a 100644 --- a/tslang/lib/TypeScript/LowerToLLVM.cpp +++ b/tslang/lib/TypeScript/LowerToLLVM.cpp @@ -477,10 +477,13 @@ class TypeDescriptorOpLowering : public TsLlvmPattern !MLIRTypeHelper::isSharedHandleType(descriptorType) ? std::string() : orl.getOrCreateRetainRoutine(descriptorType); + // empty but under rc, the one model that counts every block + auto shareRoutineName = orl.getOrCreateShareRoutine(descriptorType); rewriter.replaceOp(op, ch.getOrCreateTypeDescriptorName(descriptorType, name, TypeOfOpHelper::typeKindFromName(name), - releaseRoutineName, retainRoutineName, variant, + releaseRoutineName, retainRoutineName, + shareRoutineName, variant, TypeOfOpHelper::layoutIdOf(descriptorType))); return success(); @@ -520,6 +523,27 @@ class RetainOpLowering : public TsLlvmPattern } }; +// `Shared.share(v)`: the share routine of v's type (OwnershipRoutineLogic::getOrCreateShareRoutine) +// under rc; nothing under the models that do not count every block. +class MarkSharedOpLowering : public TsLlvmPattern +{ + public: + using TsLlvmPattern::TsLlvmPattern; + + LogicalResult matchAndRewrite(mlir_ts::MarkSharedOp op, Adaptor transformed, + ConversionPatternRewriter &rewriter) const final + { + if (tsLlvmContext->compileOptions.isRefCounted()) + { + OwnershipRoutineLogic orl(op, rewriter, getTypeConverter(), tsLlvmContext->compileOptions); + orl.emitShareValue(op.getValue().getType(), transformed.getValue()); + } + + rewriter.eraseOp(op); + return mlir::success(); + } +}; + // A promoted object's fields given back without freeing the object, and with `reset` the object // zeroed - a fresh one where its allocation was. class ReleaseContentsOpLowering : public TsLlvmPattern @@ -1372,7 +1396,12 @@ class SharedCountOpLowering : public TsLlvmPattern auto llvmIndexType = tch.convertType(th.getIndexType()); auto blockPtr = ch.getBlockPtrFromPayloadPtr(loc, transformed.getShared(), llvmIndexType); - auto count = rewriter.create(loc, llvmIndexType, blockPtr); + auto word = rewriter.create(loc, llvmIndexType, blockPtr); + // the count without the shared bit (Defines.h, beside HEAP_BLOCK_IMMORTAL) + auto bits = tsLlvmContext->compileOptions.sizeBits(); + auto mask = rewriter.create( + loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, llvm::APInt::getLowBitsSet(bits, bits - 1))); + auto count = rewriter.create(loc, llvmIndexType, word, mask); rewriter.replaceOpWithNewOp(op, numberType, count); return success(); } @@ -7744,10 +7773,12 @@ struct CreateBoundFunctionOpLowering : public TsLlvmPatterncompileOptions.memoryModel == MemoryModelOwn ? std::string() : orl.getOrCreateCaptureBoxRetainRoutine(captureRefType); + auto shareRoutineName = orl.getOrCreateCaptureBoxShareRoutine(captureRefType); LLVMCodeHelper ch(createBoundFunctionOp, rewriter, getTypeConverter(), tsLlvmContext->compileOptions); return ch.getOrCreateTypeDescriptorName(captureRefType, name, TypeOfOpHelper::typeKindFromName(name), - releaseRoutineName, retainRoutineName, borrows ? "b" : ""); + releaseRoutineName, retainRoutineName, shareRoutineName, + borrows ? "b" : ""); } // Under rc, a method bound to a class object counts that object like an interface over it does @@ -7781,10 +7812,11 @@ struct CreateBoundFunctionOpLowering : public TsLlvmPatterncompileOptions); auto releaseRoutineName = orl.getOrCreateReleaseRoutine(thisType); auto retainRoutineName = orl.getOrCreateRetainRoutine(thisType); + auto shareRoutineName = orl.getOrCreateShareRoutine(thisType); LLVMCodeHelper ch(createBoundFunctionOp, rewriter, getTypeConverter(), tsLlvmContext->compileOptions); return ch.getOrCreateTypeDescriptorName(thisType, name, TypeOfOpHelper::typeKindFromName(name), - releaseRoutineName, retainRoutineName); + releaseRoutineName, retainRoutineName, shareRoutineName); } LogicalResult matchAndRewrite(mlir_ts::CreateBoundFunctionOp createBoundFunctionOp, Adaptor transformed, @@ -9715,7 +9747,7 @@ void TypeScriptToLLVMLoweringPass::runOnOperation() ArrayPopOpLowering, ArrayUnshiftOpLowering, ArrayShiftOpLowering, ArraySpliceOpLowering, ArrayViewOpLowering, DeleteOpLowering, ParseFloatOpLowering, ParseIntOpLowering, IsNaNOpLowering, PrintOpLowering, ConvertFOpLowering, StoreOpLowering, SwapOpLowering, SizeOfOpLowering, TypeDescriptorOpLowering, RetainOpLowering, ReleaseOpLowering, RetainSlotOpLowering, ReleaseSlotOpLowering, RetainCellOpLowering, ReleaseCellOpLowering, ReleaseContentsOpLowering, MakeImmortalOpLowering, MarkImmortalCellOpLowering, InsertPropertyOpLowering, LengthOfOpLowering, SetLengthOfOpLowering, StringLengthOpLowering, StringResizeOpLowering, StringConcatOpLowering, StringCopyOpLowering, - SharedNewOpLowering, SharedValueRefOpLowering, SharedCountOpLowering, + SharedNewOpLowering, SharedValueRefOpLowering, SharedCountOpLowering, MarkSharedOpLowering, StringCompareOpLowering, AnyCompareOpLowering, CharToStringOpLowering, UndefOpLowering, CopyStructOpLowering, MemoryCopyOpLowering, MemoryMoveOpLowering, LoadSaveValueLowering, ThrowUnwindOpLowering, ThrowCallOpLowering, VariableOpLowering, DebugVariableOpLowering, AllocaOpLowering, InvokeOpLowering, AccessorUnwindEdgeOpLowering, InvokeHybridOpLowering, VirtualSymbolRefOpLowering, ThisVirtualSymbolRefOpLowering, ThisSymbolRefOpLowering, InterfaceSymbolRefOpLowering, diff --git a/tslang/lib/TypeScript/MLIRGenAccessCall.cpp b/tslang/lib/TypeScript/MLIRGenAccessCall.cpp index bad5d4d71..27a292776 100644 --- a/tslang/lib/TypeScript/MLIRGenAccessCall.cpp +++ b/tslang/lib/TypeScript/MLIRGenAccessCall.cpp @@ -2600,9 +2600,23 @@ namespace mlirgen return mlir::failure(); } + markAtomicInstanceShared(location, classInfo, V(newOp)); return V(newOp); } + // An @atomic class's new object, its constructor done: marked shared with everything its fields + // hold by then (ts.MarkShared). What its constructor stored went through the field barrier + // (markStoreIntoAtomicField) already; this catches what reached a field any other way. Under + // -mm=rc only, the one model whose counts the mark changes: elsewhere it would only keep the + // object off the stack. + void MLIRGenImpl::markAtomicInstanceShared(mlir::Location location, ClassInfo::TypePtr classInfo, mlir::Value instance) + { + if (compileOptions.isRefCounted() && classInfo && classInfo->isAtomicClass() && instance) + { + builder.create(location, instance); + } + } + // `new V(...)` of a @value class: a stack slot for its storage tuple, the constructor run with // `this` pointing at the slot, and the value loaded from it. Nothing is allocated, and nothing // keeps the slot: what `new` gives is a copy. @@ -3104,6 +3118,8 @@ namespace mlirgen auto result = mlirGenCallConstructor(location, classInfo, newOp, operands, false, genContext, constructorName); EXIT_IF_FAILED(result) } + + markAtomicInstanceShared(location, classInfo, newOp); } return newOp; diff --git a/tslang/lib/TypeScript/MLIRGenClasses.cpp b/tslang/lib/TypeScript/MLIRGenClasses.cpp index 9855ad55e..bd82329e3 100644 --- a/tslang/lib/TypeScript/MLIRGenClasses.cpp +++ b/tslang/lib/TypeScript/MLIRGenClasses.cpp @@ -460,6 +460,11 @@ namespace mlirgen { newClassPtr->isValue = true; } + + if (name == ATOMIC_DECORATOR) + { + newClassPtr->isAtomic = true; + } }); getClassesMap().insert({namePtr, newClassPtr}); diff --git a/tslang/lib/TypeScript/MLIRGenExpressions.cpp b/tslang/lib/TypeScript/MLIRGenExpressions.cpp index 82191a919..9bd5b88eb 100644 --- a/tslang/lib/TypeScript/MLIRGenExpressions.cpp +++ b/tslang/lib/TypeScript/MLIRGenExpressions.cpp @@ -1221,6 +1221,15 @@ namespace mlirgen { return mlirGenSharedCount(location, callExpression, genContext); } + + // `Shared.share(v)`: v, and every block it reaches, may be used by other threads + if (access->expression == SyntaxKind::Identifier + && MLIRHelper::getName(access->expression.as()) == "Shared" + && MLIRHelper::getName(access->name) == "share" + && !resolveIdentifier(location, "Shared", genContext)) + { + return mlirGenSharedShare(location, callExpression, genContext); + } } auto result = mlirGen(callExpr, genContext); @@ -1598,6 +1607,27 @@ namespace mlirgen return V(builder.create(location, getNumberType(), handle)); } + // `Shared.share(v)`: sets the shared bit of every counted block `v` reaches (ts.MarkShared), so + // that from here on their counts change atomically and other threads may hold them. Any value: + // one that owns no heap memory has nothing to mark. Returns nothing. + ValueOrLogicalResult MLIRGenImpl::mlirGenSharedShare(mlir::Location location, CallExpression callExpression, + const GenContext &genContext) + { + if (callExpression->arguments.size() != 1) + { + emitError(location, "Shared.share(value) takes one argument"); + return mlir::failure(); + } + + auto result = mlirGen(callExpression->arguments.front(), genContext); + EXIT_IF_FAILED_OR_NO_VALUE(result) + auto value = V(result); + + builder.create(location, value); + + return mlir::success(); + } + ValueOrLogicalResult MLIRGenImpl::mlirGen(NewExpression newExpression, const GenContext &genContext) { auto location = loc(newExpression); diff --git a/tslang/lib/TypeScript/MLIRGenImpl.h b/tslang/lib/TypeScript/MLIRGenImpl.h index e9ca4621f..4f7e5343e 100644 --- a/tslang/lib/TypeScript/MLIRGenImpl.h +++ b/tslang/lib/TypeScript/MLIRGenImpl.h @@ -7005,6 +7005,36 @@ class MLIRGenImpl return mlirGenSaveLogicOneItem(location, fieldValue, savingValue, genContext); } + // The field barrier of an @atomic class: a store through a field of an object whose static type is + // an @atomic class (or extends one) marks the value shared before it lands (ts.MarkShared), so + // nothing another thread can reach through the object is counted plainly. Under -mm=rc only. A + // store through a type that is not @atomic - a base class of one - is not seen. + void markStoreIntoAtomicField(mlir::Location location, mlir::Value storeReference, mlir::Value savingValue) + { + if (!compileOptions.isRefCounted()) + { + return; + } + + auto propertyRefOp = storeReference.getDefiningOp(); + if (!propertyRefOp) + { + return; + } + + auto classType = dyn_cast(propertyRefOp.getObjectRef().getType()); + if (!classType) + { + return; + } + + auto classInfo = getClassInfoByFullName(classType.getName().getValue()); + if (classInfo && classInfo->isAtomicClass()) + { + builder.create(location, savingValue); + } + } + ValueOrLogicalResult mlirGenSaveLogicOneItem(mlir::Location location, mlir::Value leftExpressionValue, mlir::Value rightExpressionValue, const GenContext &genContext) { @@ -7124,6 +7154,10 @@ class MLIRGenImpl return mlir::success(); } + // A field of an @atomic class: what it is given may be read by other threads from here, + // so it is marked shared first (ClassInfo::isAtomic) + markStoreIntoAtomicField(location, storeReference, savingValue); + // Overwriting owning storage hands the count over: the incoming value gains an // owner and the outgoing one loses it. Retaining first is what makes `x = x` safe - // releasing first could drop the last reference and free the value about to be @@ -10895,6 +10929,8 @@ class MLIRGenImpl ValueOrLogicalResult mlirGenNewShared(mlir::Location location, NewExpression newExpression, const GenContext &genContext); ValueOrLogicalResult mlirGenSharedCount(mlir::Location location, CallExpression callExpression, const GenContext &genContext); + ValueOrLogicalResult mlirGenSharedShare(mlir::Location location, CallExpression callExpression, const GenContext &genContext); + void markAtomicInstanceShared(mlir::Location location, ClassInfo::TypePtr classInfo, mlir::Value instance); mlir::LogicalResult mlirGen(DeleteExpression deleteExpression, const GenContext &genContext); diff --git a/tslang/lib/TypeScript/OwnershipInferenceMoves.cpp b/tslang/lib/TypeScript/OwnershipInferenceMoves.cpp index beaaf6533..b0f5df7d1 100644 --- a/tslang/lib/TypeScript/OwnershipInferenceMoves.cpp +++ b/tslang/lib/TypeScript/OwnershipInferenceMoves.cpp @@ -420,6 +420,12 @@ namespace own_inference return true; } + // `Shared.share(v)` sets bits in the headers of what `v` reaches, and keeps nothing + if (mlir::isa(user)) + { + return true; + } + // `--di`'s record of a folded const's value for the debugger: names it, keeps nothing if (mlir::isa(user)) { diff --git a/tslang/lib/TypeScript/StackEscapeFacts.h b/tslang/lib/TypeScript/StackEscapeFacts.h index 9df58e127..ed3f75354 100644 --- a/tslang/lib/TypeScript/StackEscapeFacts.h +++ b/tslang/lib/TypeScript/StackEscapeFacts.h @@ -616,6 +616,8 @@ class StackEscapeFacts return use.getOperandNumber() == 0 ? fieldRefEscapes(elementRefOp.getResult()) : "unclassified"; }) .Case([&](auto) { return ""; }) + // `Shared.share(v)`: other threads may hold it from here, past any frame + .Case([&](auto) { return "shared with other threads"; }) .Case([&](auto) { return ""; }) .Case([&](auto castOp) -> llvm::StringRef { // a view of the same object - a boxed literal seen as an object, a class as its diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index 48cbf9163..737fb0e90 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -3158,6 +3158,37 @@ if (NOT WIN32 OR TSLANG_HAVE_SHARED_GC) -P "${CMAKE_CURRENT_SOURCE_DIR}/foreign-thread-gc.cmake") endif() +# The same programs as a few of the -mm=rc tests above with --atomic-rc: the count changes are atomic +# read-modify-writes and the last reference takes an acquire fence, and a program single-threaded +# or not has to behave the same. +foreach(arc_test 00strings 00array4_push_pop 00any 00owned_closures 00interface 00shared_global_field_argument 00shared_share) + tslang_add_test(NAME test-jit-rc-atomic-${arc_test} COMMAND test-runner -jit -mm=rc -atomic-rc "${PROJECT_SOURCE_DIR}/test/tester/tests/${arc_test}.ts") + tslang_add_test(NAME test-compile-rc-atomic-${arc_test} COMMAND test-runner -mm=rc -atomic-rc "${PROJECT_SOURCE_DIR}/test/tester/tests/${arc_test}.ts") +endforeach() + +# Shared.share and @atomic class on one thread: marking ends on a cycle and leaves a literal alone, +# and a marked block counts and frees as before. Not -mm=own, which rejects the cycle it builds. +foreach(share_mm gc rc none) + tslang_add_test(NAME test-jit-${share_mm}-00-shared-share COMMAND test-runner -jit -mm=${share_mm} "${PROJECT_SOURCE_DIR}/test/tester/tests/00shared_share.ts") + tslang_add_test(NAME test-compile-${share_mm}-00-shared-share COMMAND test-runner -mm=${share_mm} "${PROJECT_SOURCE_DIR}/test/tester/tests/00shared_share.ts") +endforeach() + +# A -mm=rc shared library called from a host's threads at once, its objects shared by --atomic-rc, +# Shared.share or @atomic - see atomic-rc.cmake. +add_executable(atomic-rc-host atomic-rc-host.cpp) +if (NOT WIN32) + target_link_libraries(atomic-rc-host PRIVATE ${CMAKE_DL_LIBS} pthread) +endif() +add_test(NAME test-compile-atomic-rc + COMMAND ${CMAKE_COMMAND} + "-DTSLANG=$" + "-DHOST=$" + "-DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}/atomic-rc" + "-DLLVM_LIB=${LLVM_LIBRARY_DIR}" + "-DTSLANG_LIB=${CMAKE_BINARY_DIR}/lib" + "-DDEFAULT_LIB_CANDIDATES=${PROJECT_SOURCE_DIR}/../TypeScriptCompilerDefaultLib/__build;${PROJECT_SOURCE_DIR}/../../TypeScriptCompilerDefaultLib/__build" + -P "${CMAKE_CURRENT_SOURCE_DIR}/atomic-rc.cmake") + # One collector per process with the real default library in it - see defaultlib-collector.cmake. # Skipped (not failed) when no default library has been built; the release workflows run them after # building one. The sources live outside tests/: without the default library they do not compile, diff --git a/tslang/test/tester/atomic-rc-host.cpp b/tslang/test/tester/atomic-rc-host.cpp new file mode 100644 index 000000000..de6189599 --- /dev/null +++ b/tslang/test/tester/atomic-rc-host.cpp @@ -0,0 +1,85 @@ +// A host that loads a `-mm=rc` shared library and calls into it from several threads at once, all +// of them taking and dropping references to the same object. See atomic-rc.cmake. +// +// atomic-rc-host +// +// Each thread calls churn(calls), which takes and drops references to the library's one +// shared object. Afterwards the object's count has to be what it was before. Exits 0 when it is, +// 2 when it is not. + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +using ChurnFn = double (*)(double); +using CountFn = double (*)(); + +int main(int argc, char **argv) +{ + if (argc < 4) + { + std::printf("usage: %s \n", argv[0]); + return 1; + } + + auto threads = std::atoi(argv[2]); + auto calls = static_cast(std::atoi(argv[3])); + +#ifdef _WIN32 + auto library = LoadLibraryExA(argv[1], nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); + if (!library) + { + std::printf("LoadLibrary '%s' failed: %lu\n", argv[1], GetLastError()); + return 1; + } + + auto churn = reinterpret_cast(GetProcAddress(library, "churn")); + auto count = reinterpret_cast(GetProcAddress(library, "count")); +#else + auto library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!library) + { + std::printf("dlopen: %s\n", dlerror()); + return 1; + } + + auto churn = reinterpret_cast(dlsym(library, "churn")); + auto count = reinterpret_cast(dlsym(library, "count")); +#endif + if (!churn || !count) + { + std::printf("'%s' has no 'churn' or no 'count'\n", argv[1]); + return 1; + } + + auto before = count(); + + std::vector pool; + for (int id = 0; id < threads; id++) + { + pool.emplace_back([&]() { + if (churn(calls) != calls) + { + std::printf("churn read the shared object wrong\n"); + std::fflush(stdout); + std::exit(3); + } + }); + } + + for (auto &thread : pool) + { + thread.join(); + } + + auto after = count(); + std::printf("%d threads, %g calls each: count %g before, %g after\n", threads, calls, before, after); + return after == before ? 0 : 2; +} diff --git a/tslang/test/tester/atomic-rc.cmake b/tslang/test/tester/atomic-rc.cmake new file mode 100644 index 000000000..28d483795 --- /dev/null +++ b/tslang/test/tester/atomic-rc.cmake @@ -0,0 +1,265 @@ +# A `-mm=rc` shared library called from several threads at once (atomic-rc-host.cpp), every one of +# them taking and dropping references to the same objects: built with --atomic-rc, or with the +# objects marked shared by `Shared.share` once at load. +# +# Under plain `rc` a count change is a load, an add and a store, so two threads changing the same +# count at once lose one of the changes: the count drifts, and the object is freed while it is +# still held or never freed at all. --atomic-rc makes each change one atomic read-modify-write; +# `Shared.share(v)` sets a bit in the header of every block v reaches, and only those blocks' +# counts change atomically. Either way the count after the threads are done has to be the count +# before, exactly. +# +# The same library built without the flag is run too, but only reported: losing an update is a +# race, so its absence proves nothing, and a drift there is what the flag is for, not a failure. +# Seeing it drift is what shows this test can see the race at all. +# +# Then, when a default library has been built (DEFAULT_LIB_CANDIDATES, DEFAULT_LIB_PATH), a library +# linked against its DLL: the shared objects are a RegExp and strings the default library made, and +# RegExp.test runs in its DLL on all threads at once. The descriptor whose share routine marks them +# is the default library's, so this is where a library built before the share slot would show. +# Under --atomic-rc alone the default library's own code still counts an unmarked block plainly, +# so that build is only reported. + +cmake_minimum_required(VERSION 3.17.3) + +foreach(var TSLANG HOST WORK_DIR TSLANG_LIB LLVM_LIB DEFAULT_LIB_CANDIDATES) + if(NOT DEFINED ${var}) + message(FATAL_ERROR "${var} is required") + endif() +endforeach() + +file(REMOVE_RECURSE "${WORK_DIR}") +file(MAKE_DIRECTORY "${WORK_DIR}") + +set(ENV{TSLANG_LIB_PATH} "") + +file(WRITE "${WORK_DIR}/shared_churn.ts" [=[ +interface HasV { + v: number; +} + +class Child { + v: number = 1; +} + +function makeReader(c: Child) { + return () => c.v; +} + +class Node { + v: number = 1; + // a literal: immortal, which marking has to leave alone (its header may be read-only) + tag = "node"; + child = new Child(); + children: Child[] = [new Child(), new Child()]; + // reached only through a descriptor: what an `any`, an interface, a union and a closure hold + boxed: any = new Child(); + viewed: HasV = new Child(); + either: Child | string = new Child(); + reader = makeReader(new Child()); +} + +class Box { + s: Shared; + child: Child; + children: Child[]; + tag: string; + boxed: any; + viewed: HasV; + either: Child | string; + reader: () => number; + + constructor(s: Shared) { + const node = s.value; + this.s = s; + this.child = node.child; + this.children = node.children; + this.tag = node.tag; + this.boxed = node.boxed; + this.viewed = node.viewed; + this.either = node.either; + this.reader = node.reader; + } +} + +// The root every thread starts from: written here, while one thread runs, and only read after. +// Without @atomic or Shared.share its blocks are counted plainly. +ATOMIC_IT class Holder { + s = new Shared(new Node()); + late: Child | undefined; +} + +const H = new Holder(); +// stored after the constructor: an @atomic class's field barrier marks it +H.late = new Child(); +SHARE_IT + +// every Box takes a reference to each of them, and gives it back when it goes +export function churn(n: number): number { + let sum: number = 0; + for (let i = 0; i < n; i++) { + const box = new Box(H.s); + const late = H.late; + sum += box.s.value.v * box.child.v * box.children[1].v * box.viewed.v * box.reader() * (late ? late.v : 0); + } + + return sum; +} + +export function count(): number { + return Shared.count(H.s); +} +]=]) + +set(libs "--tslang-lib-path=${TSLANG_LIB}" "--llvm-lib-path=${LLVM_LIB}") +set(default_lib_option --no-default-lib) + +if(WIN32) + set(prefix "") + set(suffix ".dll") + set(pic "") +else() + set(prefix "lib") + set(suffix ".so") + set(pic "-relocation-model=pic") +endif() + +file(READ "${WORK_DIR}/shared_churn.ts" churn_source) +string(REPLACE "ATOMIC_IT" "" churn_source_plain "${churn_source}") +string(REPLACE "SHARE_IT" "" unmarked "${churn_source_plain}") +string(REPLACE "SHARE_IT" "Shared.share(H);" marked "${churn_source_plain}") +string(REPLACE "ATOMIC_IT" "@atomic" churn_source_atomic "${churn_source}") +string(REPLACE "SHARE_IT" "" atomic_class "${churn_source_atomic}") +file(WRITE "${WORK_DIR}/unmarked.ts" "${unmarked}") +file(WRITE "${WORK_DIR}/marked.ts" "${marked}") +file(WRITE "${WORK_DIR}/atomic_class.ts" "${atomic_class}") + +function(build_and_run name source expect_exact) + set(library "${WORK_DIR}/${prefix}${name}${suffix}") + execute_process(COMMAND "${TSLANG}" --emit=dll --opt -mm=rc ${ARGN} ${default_lib_option} ${pic} ${libs} + ${source} -o "${library}" + WORKING_DIRECTORY "${WORK_DIR}" + OUTPUT_VARIABLE out + ERROR_VARIABLE err + RESULT_VARIABLE status) + if(NOT status EQUAL 0) + message(FATAL_ERROR "--emit=dll ${name}: exit ${status}\n${out}\n${err}") + endif() + + execute_process(COMMAND "${HOST}" "${library}" 8 200000 + WORKING_DIRECTORY "${WORK_DIR}" + OUTPUT_VARIABLE out + ERROR_VARIABLE err + RESULT_VARIABLE status + TIMEOUT 240) + if(expect_exact) + if(NOT status EQUAL 0) + message(FATAL_ERROR "${name}: the count after the threads is not the count before: exit ${status}\n${out}\n${err}") + endif() + message(STATUS "${name}: ${out}") + else() + message(STATUS "${name} (reported only, exit ${status}): ${out}") + endif() +endfunction() + +# every count atomic +build_and_run(atomic unmarked.ts TRUE --atomic-rc) +# Shared.share(H) at load: everything H reaches is marked, and only those counts change atomically +build_and_run(marked marked.ts TRUE) +# `@atomic class Holder`: marked once its constructor has run, and `H.late = ...` marked by the +# field barrier +build_and_run(atomic-class atomic_class.ts TRUE) +# both: a block with the shared bit under --atomic-rc still frees at the right count +build_and_run(marked-atomic marked.ts TRUE --atomic-rc) +# neither: reported only +build_and_run(plain unmarked.ts FALSE) + +# Against the default library's DLL, when one has been built for this target and model +execute_process(COMMAND "${TSLANG}" --print-default-lib-dir=dll --opt -mm=rc + OUTPUT_VARIABLE default_lib_dll_subdir + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE status) +if(NOT status EQUAL 0 OR default_lib_dll_subdir STREQUAL "") + message(FATAL_ERROR "tslang --print-default-lib-dir=dll --opt -mm=rc: exit ${status}") +endif() + +set(default_lib "") +foreach(candidate "$ENV{DEFAULT_LIB_PATH}" ${DEFAULT_LIB_CANDIDATES}) + if(NOT candidate STREQUAL "" AND IS_DIRECTORY "${candidate}/${default_lib_dll_subdir}") + set(default_lib "${candidate}") + break() + endif() +endforeach() + +if(default_lib STREQUAL "") + message(STATUS "no default library built in ${default_lib_dll_subdir}: its DLL is not checked") + return() +endif() + +message(STATUS "default library: ${default_lib}") +set(ENV{DEFAULT_LIB_PATH} "") +set(default_lib_option "--default-lib-path=${default_lib}") +# the host loads a library's dependencies from the library's folder +file(COPY "${default_lib}/${default_lib_dll_subdir}/${prefix}TypeScriptDefaultLib${suffix}" DESTINATION "${WORK_DIR}") + +file(WRITE "${WORK_DIR}/defaultlib_churn.ts" [=[ +class Child { + v: number = 1; +} + +class Box { + s: Shared; + rx: Shared; + words: string[]; + + constructor(s: Shared, rx: Shared, words: string[]) { + this.s = s; + this.rx = rx; + this.words = words; + } +} + +// objects of the default library's: a RegExp, and strings split made +ATOMIC_IT class Holder { + s = new Shared(new Child()); + rx = new Shared(new RegExp("ab")); + words: string[] = "x,ab,y".split(","); +} + +const H = new Holder(); +SHARE_IT + +export function churn(n: number): number { + let sum: number = 0; + for (let i = 0; i < n; i++) { + const box = new Box(H.s, H.rx, H.words); + // in the default library's DLL, on the shared RegExp and a shared string + if (box.rx.value.test(box.words[1])) { + sum += box.s.value.v; + } + } + + return sum; +} + +export function count(): number { + return Shared.count(H.s) * 1000 + Shared.count(H.rx); +} +]=]) + +file(READ "${WORK_DIR}/defaultlib_churn.ts" churn_source) +string(REPLACE "ATOMIC_IT " "" churn_source_plain "${churn_source}") +string(REPLACE "SHARE_IT" "" unmarked "${churn_source_plain}") +string(REPLACE "SHARE_IT" "Shared.share(H);" marked "${churn_source_plain}") +string(REPLACE "ATOMIC_IT " "@atomic " churn_source_atomic "${churn_source}") +string(REPLACE "SHARE_IT" "" atomic_class "${churn_source_atomic}") +file(WRITE "${WORK_DIR}/defaultlib_unmarked.ts" "${unmarked}") +file(WRITE "${WORK_DIR}/defaultlib_marked.ts" "${marked}") +file(WRITE "${WORK_DIR}/defaultlib_atomic_class.ts" "${atomic_class}") + +build_and_run(defaultlib-marked defaultlib_marked.ts TRUE) +build_and_run(defaultlib-atomic-class defaultlib_atomic_class.ts TRUE) +build_and_run(defaultlib-marked-atomic defaultlib_marked.ts TRUE --atomic-rc) +# the default library's own count changes on these blocks stay plain: reported only +build_and_run(defaultlib-atomic defaultlib_unmarked.ts FALSE --atomic-rc) +build_and_run(defaultlib-plain defaultlib_unmarked.ts FALSE) diff --git a/tslang/test/tester/test-runner.cpp b/tslang/test/tester/test-runner.cpp index 0a535511e..1873fee79 100644 --- a/tslang/test/tester/test-runner.cpp +++ b/tslang/test/tester/test-runner.cpp @@ -102,6 +102,7 @@ auto fastMath = false; auto memoryModel = std::string(""); // `-no-stack-promotion`: build with every allocation on the heap, for comparing against the default auto noStackPromotion = false; +auto atomicRC = false; auto tslang_opt_ext = std::string(""); // `-x86`: build and run a 32-bit (i686) program. The target triple goes into tslang_opt rather @@ -157,7 +158,7 @@ void writeLibPathSettings(std::ostream &batFile) // would let whichever runner created it first decide the flags for everyone. std::string optVariantSuffix() { - return std::string(fastMath ? "fm" : "") + memoryModel + (x86 ? "x86" : "") + (noStackPromotion ? "nosp" : ""); + return std::string(fastMath ? "fm" : "") + memoryModel + (x86 ? "x86" : "") + (noStackPromotion ? "nosp" : "") + (atomicRC ? "arc" : ""); } // What tslang links into an executable (exe.cpp): mimalloc, under every memory model but gc. A @@ -821,6 +822,12 @@ void readParams(int argc, char **argv, std::vector &files) // into tslang_opt, like -mm=: every file of a multi-file test is built without it tslang_opt += " --no-stack-promotion"; } + else if (std::string(argv[index]) == "-atomic-rc") + { + atomicRC = true; + // into tslang_opt, like -mm=: every file of a multi-file test counts atomically + tslang_opt += " --atomic-rc"; + } // `-mm=gc` is accepted as well as the three that change behaviour, and it is not a no-op: // it names the default explicitly, which is what anyone comparing the models types. // It gets its own cached script like the others - the suffix is what keeps two runners diff --git a/tslang/test/tester/tests/00shared_share.ts b/tslang/test/tester/tests/00shared_share.ts new file mode 100644 index 000000000..3e9888ed2 --- /dev/null +++ b/tslang/test/tester/tests/00shared_share.ts @@ -0,0 +1,104 @@ +// `Shared.share(v)` marks every block v reaches as shared between threads (ts.MarkShared), and an +// `@atomic class` marks its new objects and what its fields are given. On one thread nothing +// changes but how the counts are changed, so this checks that marking ends - on a cycle, on a +// literal whose header may be read-only - and that a marked block still counts and frees as +// before: Shared.count reads the count without the bit, and the program runs to the end under +// rc, where a count the mark disturbed would free a block in use. gc and none count nothing and +// report -1. + +interface HasV { + v: number; +} + +class Leaf { + v: number = 1; +} + +class Ring { + next: Ring | undefined; + name: string; + + constructor(name: string) { + this.name = name; + } +} + +function makeReader(leaf: Leaf) { + return () => leaf.v; +} + +class Bag { + // a literal: immortal, left alone + tag = "bag"; + leaves: Leaf[] = [new Leaf(), new Leaf()]; + names: string[] = ["a", "b"]; + boxed: any = new Leaf(); + viewed: HasV = new Leaf(); + either: Leaf | string = new Leaf(); + maybe: Leaf | undefined = new Leaf(); + reader = makeReader(new Leaf()); + handle = new Shared(new Leaf()); +} + +@atomic +class Counter { + hits: number = 0; + last: Leaf | undefined; + seen: Leaf[] = []; +} + +// a subclass of an @atomic class is @atomic too +class NamedCounter extends Counter { + label: string = "counter"; +} + +function countIs(s: Shared, expected: number) { + const c = Shared.count(s); + return c < 0 || c == expected; +} + +function main() { + // a cycle: the walk stops at the block it marked first + const a = new Ring("a"); + const b = new Ring("b"); + a.next = b; + b.next = a; + Shared.share(a); + Shared.share(a); + assert(a.next.name == "b" && b.next.name == "a", "a marked cycle reads as before"); + b.next = undefined; + + // every kind of field, through descriptors too + const bag = new Bag(); + assert(countIs(bag.handle, 1), "a handle's count before the mark"); + Shared.share(bag); + assert(countIs(bag.handle, 1), "the mark leaves the count alone"); + const extra = bag.handle; + assert(countIs(extra, 2), "a marked handle still counts up"); + assert(bag.leaves[1].v == 1 && bag.names[0] == "a" && bag.viewed.v == 1 && bag.reader() == 1, + "a marked object reads as before"); + assert(bag.tag == "bag", "a literal is left alone"); + + // values that own nothing, and a value marked twice + Shared.share(1); + Shared.share("literal"); + Shared.share(bag.leaves); + Shared.share(bag.leaves); + + // a marked block is still freed: replaced, its last reference goes + bag.leaves = [new Leaf()]; + bag.maybe = undefined; + assert(bag.leaves.length == 1, "a field of a marked object takes a new value"); + + // an @atomic class: marked when made, and each value its fields are given marked as it lands + const counter = new NamedCounter(); + counter.hits = counter.hits + 1; + counter.last = new Leaf(); + // not marked: the barrier sees field stores, not an element added to a field's array. This + // checks only that the program runs on + counter.seen.push(new Leaf()); + counter.last = undefined; + assert(counter.hits == 1 && counter.seen.length == 1 && counter.label == "counter", "an @atomic object reads as before"); + + print("done."); +} diff --git a/tslang/tslang/jitcache.cpp b/tslang/tslang/jitcache.cpp index 760b1ec62..68c675b8e 100644 --- a/tslang/tslang/jitcache.cpp +++ b/tslang/tslang/jitcache.cpp @@ -360,7 +360,7 @@ class JitUnitBuilder std::string key; llvm::raw_string_ostream os(key); os << (isMain ? "main " + mainFuncName.getValue() : std::string("import")) << "\n"; - os << "mm=" << compileOptions.memoryModel << " opt=" << enableOpt.getValue() << " opt_level=" << optLevel.getValue() + os << "mm=" << compileOptions.memoryModel << " atomic-rc=" << compileOptions.atomicRC << " opt=" << enableOpt.getValue() << " opt_level=" << optLevel.getValue() << " size_level=" << sizeLevel.getValue() << " di=" << compileOptions.generateDebugInfo << " lldb=" << compileOptions.lldbDebugInfo << " no-default-lib=" << compileOptions.noDefaultLib << " builtins=" << compileOptions.enableBuiltins << " strict-null-checks=" << compileOptions.strictNullChecks diff --git a/tslang/tslang/jitnative.cpp b/tslang/tslang/jitnative.cpp index bb4aef58c..eaa40cb38 100644 --- a/tslang/tslang/jitnative.cpp +++ b/tslang/tslang/jitnative.cpp @@ -221,7 +221,7 @@ int linkJitUnits(const char *argv0, llvm::ArrayRef units, CompileOption } os << "crt=" << (debugCRT ? "debug" : "release") << " di=" << compileOptions.generateDebugInfo - << " mm=" << compileOptions.memoryModel << " no-default-lib=" << compileOptions.noDefaultLib << "\n"; + << " mm=" << compileOptions.memoryModel << " atomic-rc=" << compileOptions.atomicRC << " no-default-lib=" << compileOptions.noDefaultLib << "\n"; os << "default-lib=" << (compileOptions.noDefaultLib ? std::string() : getDefaultLibBinary(/*shared=*/false, compileOptions, tm.getTargetTriple())) << "\n"; diff --git a/tslang/tslang/opts.cpp b/tslang/tslang/opts.cpp index b2c12540b..f296251b1 100644 --- a/tslang/tslang/opts.cpp +++ b/tslang/tslang/opts.cpp @@ -17,6 +17,7 @@ namespace cl = llvm::cl; extern cl::opt inputFilename; extern cl::opt emitAction; extern cl::opt memoryModelOpt; +extern cl::opt atomicRCOpt; extern cl::opt disableWarnings; extern cl::opt generateDebugInfo; extern cl::opt lldbDebugInfo; @@ -47,6 +48,7 @@ CompileOptions prepareOptions() CompileOptions compileOptions; compileOptions.isJit = emitAction.getValue() == Action::RunJIT; compileOptions.memoryModel = memoryModelOpt.getValue(); + compileOptions.atomicRC = atomicRCOpt.getValue(); compileOptions.enableBuiltins = enableBuiltins.getValue(); compileOptions.noDefaultLib = noDefaultLib.getValue(); compileOptions.disableWarnings = disableWarnings.getValue(); diff --git a/tslang/tslang/tslang.cpp b/tslang/tslang/tslang.cpp index 2a22b981b..5fbeb5da4 100644 --- a/tslang/tslang/tslang.cpp +++ b/tslang/tslang/tslang.cpp @@ -145,6 +145,7 @@ cl::opt memoryModelOpt("mm", cl::desc("Memory management of co cl::values(clEnumValN(MemoryModelNone, "none", "no reclamation, leak everything")), cl::values(clEnumValN(MemoryModelOwn, "own", "single ownership inferred at compile time, no counting (in development; ownership that cannot be proven is a compile error)")), cl::init(MemoryModelGC), cl::cat(TypeScriptCompilerCategory)); +cl::opt atomicRCOpt("atomic-rc", cl::desc("-mm=rc: change reference counts with atomic operations, so objects can be shared between threads"), cl::cat(TypeScriptCompilerCategory)); cl::opt disableWarnings("nowarn", cl::desc("Disable Warnings"), cl::cat(TypeScriptCompilerCategory)); cl::opt verifyOwnership("verify-ownership", cl::desc("Check that every slot taking a reference gives it back on every path out of the function, unwind paths included"), cl::cat(TypeScriptCompilerCategory)); cl::opt noStackPromotion("no-stack-promotion", cl::desc("Keep every allocation on the heap; by default one that provably never outlives its function goes on the stack"), cl::cat(TypeScriptCompilerCategory)); @@ -447,6 +448,13 @@ int main(int argc, char **argv) cl::ParseCommandLineOptions(argc, argv, "TypeScript native compiler\n"); + // only `rc` keeps counts for the flag to make atomic + if (atomicRCOpt && memoryModelOpt != MemoryModelRC) + { + llvm::WithColor::error(llvm::errs(), "tslang") << "--atomic-rc needs -mm=rc\n"; + return -1; + } + // Before program arguments existed a second positional was an error; keep it one wherever // nothing would receive it if (!programArgs.empty() && (emitAction != Action::RunJIT || newVSCodeFolder || newCMakeFolder || installDefaultLibCmd))