diff --git a/README.md b/README.md index 7d6ed7e87..5f675cd8b 100644 --- a/README.md +++ b/README.md @@ -760,31 +760,30 @@ what to do about it, and **[docs/own-memory-model.md](docs/own-memory-model.md)* ### Performance The ray tracer benchmark in [tslang/benchmarks/raytrace](tslang/benchmarks/raytrace/README.md) -(`raytrace.ts` at 512x512, median of 7 runs; the `-mm=own` row measured 2026-10-23, the other rows -2026-10-21) against Rust and C++ ports of the same -code, each written twice: with plain value vectors and colors, and with every object -reference-counted on the heap the way tslang allocates them: - -| build | time | peak memory | vs C++ values (clang -O3) | -| --- | --- | --- | --- | -| `-mm=gc` | 1298 ms | 6 MB | 26x slower | -| `-mm=rc` | 599 ms | 4 MB | 12x slower | -| `-mm=none` | 703 ms | 845 MB | 14x slower | -| `-mm=own` (2026-10-23) | 628 ms | 4.5 MB | 12x slower | -| C++, values (clang / MSVC) | 51 / 69 ms | | | -| Rust, values (`Copy` vectors and colors) | 55 ms | 4 MB | 1.1x slower | -| Rust, every object in an `Rc` | 492 ms | 4 MB | 9.7x slower | -| C++, every object in a `shared_ptr` (clang / MSVC) | 830 / 899 ms | | 16x slower | - -Almost all of the gap is allocation: every vector and color the program computes is a heap object. -Rust that allocates the same way is only 1.2x faster than `-mm=rc`, and C++ with `shared_ptr` -(atomic counts) is 1.4x slower than it. `-mm=none` is slower than -`-mm=rc` because it never frees and keeps touching fresh pages. Stack promotion makes no measurable -difference here yet: the hot allocations are returned from functions or captured by closures. -`-mm=own` builds the benchmark's copy with two source edits (the shared surfaces are `Shared` -handles, and one method returns a fresh vector instead of its own field). It runs as fast as -`-mm=rc` in as little memory, counting only those two handles (its row measured 2026-10-23, when -`-mm=rc` took 638 ms and peaked at 4.5 MB). +(`raytrace.ts` at 512x512; release build, Windows 11 x64, median of 7 runs; the tslang rows measured +2026-10-31, the Rust and C++ rows 2026-10-21 on the same machine) against Rust and C++ ports of the +same code. The benchmark's copy marks `Vector` and `Color` as `@value` classes and `Ray` and +`Intersection` as `@value` interfaces, so they are structs passed by value rather than heap +objects; `-mm=own` builds `rtbench_own.ts`, the same program with the shared surfaces as +`Shared` handles. + +| build | time | vs Rust (values) | +| --- | --- | --- | +| `-mm=gc` | 71 ms | 1.3x slower | +| `-mm=none` | 70 ms | 1.3x slower | +| `-mm=own` | 77 ms | 1.4x slower | +| `-mm=rc` | 110 ms | 2.0x slower | +| Rust, values (`Copy` vectors and colors) | 55 ms | | +| C++, values (clang -O3 / MSVC /O2) | 51 / 69 ms | | +| Rust, every object in an `Rc` | 492 ms | 8.9x slower | +| C++, every object in a `shared_ptr` (clang / MSVC) | 830 / 899 ms | 15x slower | + +On 2026-10-21, before `@value`, every vector and color the program computed was a heap object and +the same benchmark took 1298 ms under `-mm=gc` and 599 ms under `-mm=rc`; `@value` took it to 255 +and 172 ms. The rest of the gain came from capturing parameters by copy instead of in heap cells, +calling methods nothing overrides directly, and testing a union for null without a tag. `-mm=rc` +still pays for retains and releases in the inner loops. The benchmark's README has every step and +the earlier results. ### `-mm=own` diff --git a/tslang/include/TypeScript/MLIRLogic/MLIRHelper.h b/tslang/include/TypeScript/MLIRLogic/MLIRHelper.h index 4eabb42eb..805e8eb9a 100644 --- a/tslang/include/TypeScript/MLIRLogic/MLIRHelper.h +++ b/tslang/include/TypeScript/MLIRLogic/MLIRHelper.h @@ -20,17 +20,60 @@ namespace mlir_ts = mlir::typescript; namespace typescript { +// A wide string of the parser holds UTF-16 units on every platform - stows() reads a source with +// codecvt_utf8_utf16, as TypeScript counts a string in UTF-16 units - so a character outside the BMP +// is a surrogate pair even where wchar_t has 32 bits (Linux). llvm::convertWideToUTF8 takes a 32-bit +// wchar_t string for UTF-32, where a surrogate is invalid, and gave "" for a literal such as "😀". static std::string convertWideToUTF8(const std::wstring &ws) { std::string s; - llvm::convertWideToUTF8(ws, s); + if constexpr (sizeof(wchar_t) == sizeof(llvm::UTF16)) + { + llvm::convertWideToUTF8(ws, s); + } + else + { + llvm::SmallVector units; + units.reserve(ws.size()); + for (auto c : ws) + { + auto code = static_cast(c); + if (code > 0xFFFF && code <= 0x10FFFF) + { + // a code point made elsewhere (not by stows) - its surrogate pair + code -= 0x10000; + units.push_back(static_cast(0xD800 + (code >> 10))); + units.push_back(static_cast(0xDC00 + (code & 0x3FF))); + } + else + { + units.push_back(static_cast(code)); + } + } + + llvm::convertUTF16ToUTF8String(units, s); + } + return s; } +// the other way, UTF-16 units again, as stows() makes them static std::wstring convertUTF8toWide(const std::string &s) { std::wstring ws; - llvm::ConvertUTF8toWide(s, ws); + if constexpr (sizeof(wchar_t) == sizeof(llvm::UTF16)) + { + llvm::ConvertUTF8toWide(s, ws); + } + else + { + llvm::SmallVector units; + if (llvm::convertUTF8ToUTF16String(s, units)) + { + ws.assign(units.begin(), units.end()); + } + } + return ws; } diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index 3cdb687e9..77a0ca3a2 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -551,6 +551,7 @@ tslang_add_test(NAME test-compile-00-owned-closures COMMAND test-runner "${PROJE tslang_add_test(NAME test-compile-00-owned-any-boxing COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00owned_any_boxing.ts") tslang_add_test(NAME test-compile-00-owned-strings COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00owned_strings.ts") tslang_add_test(NAME test-compile-00-string-length-resize COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00string_length_resize.ts") +tslang_add_test(NAME test-compile-00-string-non-bmp COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00string_non_bmp.ts") tslang_add_test(NAME test-compile-00-narrow-array-null COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00narrow_array_null.ts") tslang_add_test(NAME test-compile-00-array-length-grow-zeroed COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00array_length_grow_zeroed.ts") tslang_add_test(NAME test-compile-00-rest-args-pack-owned COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/tests/00rest_args_pack_owned.ts") @@ -1214,6 +1215,7 @@ tslang_add_test(NAME test-jit-00-owned-closures COMMAND test-runner -jit "${PROJ tslang_add_test(NAME test-jit-00-owned-any-boxing COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00owned_any_boxing.ts") tslang_add_test(NAME test-jit-00-owned-strings COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00owned_strings.ts") tslang_add_test(NAME test-jit-00-string-length-resize COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00string_length_resize.ts") +tslang_add_test(NAME test-jit-00-string-non-bmp COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00string_non_bmp.ts") tslang_add_test(NAME test-jit-00-narrow-array-null COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/tests/00narrow_array_null.ts") # own sees each branch's array of its literal as fresh tslang_add_test(NAME test-jit-own-00-narrow-array-null COMMAND test-runner -jit -mm=own "${PROJECT_SOURCE_DIR}/test/tester/tests/00narrow_array_null.ts") diff --git a/tslang/test/tester/tests/00string_non_bmp.ts b/tslang/test/tester/tests/00string_non_bmp.ts new file mode 100644 index 000000000..820c3746b --- /dev/null +++ b/tslang/test/tester/tests/00string_non_bmp.ts @@ -0,0 +1,21 @@ +// a character outside the BMP in a string literal: the parser holds it as a UTF-16 surrogate pair, +// and on Linux (32-bit wchar_t) the literal came out as "" - strings are UTF-8, so it is 4 bytes +const globalSmile = "😀"; + +function bytes(s: string) { + return s.length; +} + +function main() { + const smile = "😀"; + assert(smile.length == 4, "a literal outside the BMP is four bytes"); + assert(bytes(globalSmile) == 4, "a global literal outside the BMP"); + assert(smile == "\u{1F600}", "the same as a code point escape"); + assert(smile == "😀", "the same as a surrogate pair escape"); + assert(`<${smile}>`.length == 6, "in a template"); + assert(`😀`.length == 4, "a template literal outside the BMP"); + assert("a😀b".length == 6, "between other characters"); + assert("é€😀".length == 9, "two, three and four bytes"); + + print(smile, "done."); +} diff --git a/tslang/ts-new-parser/scanner.cpp b/tslang/ts-new-parser/scanner.cpp index d446feb08..13a55809c 100644 --- a/tslang/ts-new-parser/scanner.cpp +++ b/tslang/ts-new-parser/scanner.cpp @@ -3627,7 +3627,16 @@ auto Scanner::charSize(CharacterCodes ch) -> number auto Scanner::utf16EncodeAsStringFallback(number codePoint) -> string { debug(0x0 <= codePoint && codePoint <= 0x10FFFF); - return fromCharCode(codePoint); + if (codePoint <= 0xFFFF) + { + return fromCharCode(codePoint); + } + + // past the BMP: a surrogate pair, as every other string of the parser holds such a character + // (fromCharCode alone kept the low 16 bits where wchar_t has 16, so "\u{1F600}" was U+F600) + auto codeUnit1 = (codePoint - 0x10000) / 0x400 + 0xD800; + auto codeUnit2 = (codePoint - 0x10000) % 0x400 + 0xDC00; + return fromCharCode(codeUnit1) + fromCharCode(codeUnit2); } /* @internal */