From 10daaf26242e1d329944d3aa85056ac16774adaf Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Thu, 5 Nov 2026 18:39:26 +0000 Subject: [PATCH] An import is discovered once: nested imports were generated 2^depth times mlirGenInclude ran a discovery pass and then codegen for every source import. Discovery works in a throwaway module with an empty emittedFiles (#371: that module must hold every declaration), so each import's own imports were generated in both passes again - an import d levels deep was generated 2^d times. powder_sandbox's checks.ts sits on a chain of eight imports: at -O3 -mm=rc it ran past 10 GB and several minutes (16 GB / 168 s at -O0). An import whose discovery has run once (discoveredImports, which survives discovery like filesInProgress) is now generated into the current module without being discovered again: its declarations are in the maps already. Every module still gets every file. checks.ts: 4.8 s / 360 MB at -O3 -mm=rc; simulation.ts's IR is unchanged except generated ids. Test: import-chain/deep_chain.ts, a chain of 20 imports with a diamond at each link (TIMEOUT 120; unfixed it ran past 150 s, fixed it compiles in 0.3 s). Co-Authored-By: Claude Opus 5.5 --- tslang/lib/TypeScript/MLIRGenImpl.h | 7 +++++++ tslang/lib/TypeScript/MLIRGenModule.cpp | 6 +++++- tslang/test/tester/CMakeLists.txt | 17 +++++++++++++++++ tslang/test/tester/import-chain/chain_01.ts | 3 +++ tslang/test/tester/import-chain/chain_02.ts | 4 ++++ tslang/test/tester/import-chain/chain_03.ts | 5 +++++ tslang/test/tester/import-chain/chain_04.ts | 5 +++++ tslang/test/tester/import-chain/chain_05.ts | 5 +++++ tslang/test/tester/import-chain/chain_06.ts | 5 +++++ tslang/test/tester/import-chain/chain_07.ts | 5 +++++ tslang/test/tester/import-chain/chain_08.ts | 5 +++++ tslang/test/tester/import-chain/chain_09.ts | 5 +++++ tslang/test/tester/import-chain/chain_10.ts | 5 +++++ tslang/test/tester/import-chain/chain_11.ts | 5 +++++ tslang/test/tester/import-chain/chain_12.ts | 5 +++++ tslang/test/tester/import-chain/chain_13.ts | 5 +++++ tslang/test/tester/import-chain/chain_14.ts | 5 +++++ tslang/test/tester/import-chain/chain_15.ts | 5 +++++ tslang/test/tester/import-chain/chain_16.ts | 5 +++++ tslang/test/tester/import-chain/chain_17.ts | 5 +++++ tslang/test/tester/import-chain/chain_18.ts | 5 +++++ tslang/test/tester/import-chain/chain_19.ts | 5 +++++ tslang/test/tester/import-chain/chain_20.ts | 5 +++++ tslang/test/tester/import-chain/deep_chain.ts | 8 ++++++++ 24 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 tslang/test/tester/import-chain/chain_01.ts create mode 100644 tslang/test/tester/import-chain/chain_02.ts create mode 100644 tslang/test/tester/import-chain/chain_03.ts create mode 100644 tslang/test/tester/import-chain/chain_04.ts create mode 100644 tslang/test/tester/import-chain/chain_05.ts create mode 100644 tslang/test/tester/import-chain/chain_06.ts create mode 100644 tslang/test/tester/import-chain/chain_07.ts create mode 100644 tslang/test/tester/import-chain/chain_08.ts create mode 100644 tslang/test/tester/import-chain/chain_09.ts create mode 100644 tslang/test/tester/import-chain/chain_10.ts create mode 100644 tslang/test/tester/import-chain/chain_11.ts create mode 100644 tslang/test/tester/import-chain/chain_12.ts create mode 100644 tslang/test/tester/import-chain/chain_13.ts create mode 100644 tslang/test/tester/import-chain/chain_14.ts create mode 100644 tslang/test/tester/import-chain/chain_15.ts create mode 100644 tslang/test/tester/import-chain/chain_16.ts create mode 100644 tslang/test/tester/import-chain/chain_17.ts create mode 100644 tslang/test/tester/import-chain/chain_18.ts create mode 100644 tslang/test/tester/import-chain/chain_19.ts create mode 100644 tslang/test/tester/import-chain/chain_20.ts create mode 100644 tslang/test/tester/import-chain/deep_chain.ts diff --git a/tslang/lib/TypeScript/MLIRGenImpl.h b/tslang/lib/TypeScript/MLIRGenImpl.h index a4e8e17c7..ee7dcb8ad 100644 --- a/tslang/lib/TypeScript/MLIRGenImpl.h +++ b/tslang/lib/TypeScript/MLIRGenImpl.h @@ -16814,6 +16814,13 @@ class MLIRGenImpl // imports a) stops instead of recursing until the stack overflows. llvm::StringSet<> filesInProgress; + // Source imports whose own discovery pass has run once (mlirGenInclude). Their declarations + // are in the maps from then on, so an import generated into another module - each discovery + // module, then theModule - is generated without discovering it again. Discovered again each + // time, an import nested d deep was generated 2^d times: a chain of eight imports took 10 GB. + // Survives discovery, as filesInProgress does. + llvm::StringSet<> discoveredImports; + // The default library's files - lib.d.ts and every file it references - by file key. Its // ambient data is defined in its DLL when the program links that (see // isImportedFromSharedDefaultLib). diff --git a/tslang/lib/TypeScript/MLIRGenModule.cpp b/tslang/lib/TypeScript/MLIRGenModule.cpp index 6a7cf9b15..451f479ce 100644 --- a/tslang/lib/TypeScript/MLIRGenModule.cpp +++ b/tslang/lib/TypeScript/MLIRGenModule.cpp @@ -1423,7 +1423,10 @@ namespace mlirgen MLIRValueGuard> *> diagnosticsGuard(importDiagnostics); importDiagnostics = &diagnostics; - generated = mlir::succeeded(mlirDiscoverAllDependencies(importSource, importIncludeFiles)) && + // discovered once already (generated into an earlier module): its declarations are + // known, and discovering it again would generate every import inside it twice more + generated = (discoveredImports.contains(canonicalPath) || + mlir::succeeded(mlirDiscoverAllDependencies(importSource, importIncludeFiles))) && mlir::succeeded(mlirCodeGenModule(importSource, importIncludeFiles, false, false)); } @@ -1441,6 +1444,7 @@ namespace mlirgen emittedFiles.insert(canonicalPath); emittedFiles.insert(declSymbol); emittedFiles.insert(genericDeclSymbol); + discoveredImports.insert(canonicalPath); // the object the JIT has to load for it; after the ones it imports, which were // generated inside it. A pass generated again (discovery, then the module) finds it diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index 3b35cca7c..415648bd3 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -5139,6 +5139,23 @@ endfunction() set(reference_path_dir "${PROJECT_SOURCE_DIR}/test/tester/reference-path") tslang_add_import_tests(reference-path-import-and-reference "${reference_path_dir}" import_and_reference module_left) tslang_add_import_tests(reference-path-two-imports "${reference_path_dir}" two_imports module_left module_right) + +# An import discovered again in every module it was generated into was generated 2^depth times: +# twenty imports deep took about 1000 s, so the TIMEOUT is what fails it if that comes back. +set(import_chain_modules "") +foreach(import_chain_index RANGE 1 20) + if (import_chain_index LESS 10) + set(import_chain_index "0${import_chain_index}") + endif() + list(APPEND import_chain_modules "${PROJECT_SOURCE_DIR}/test/tester/import-chain/chain_${import_chain_index}.ts") +endforeach() +# Not -shared: a module that imports another is declared twice in the one library, chain or not. +# The JIT compiles each import into its cache by itself. +tslang_add_test(NAME test-compile-import-chain-deep + COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/import-chain/deep_chain.ts" ${import_chain_modules}) +tslang_add_test(NAME test-jit-import-chain-deep + COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/import-chain/deep_chain.ts") +set_tests_properties(test-compile-import-chain-deep test-jit-import-chain-deep PROPERTIES TIMEOUT 120) tslang_add_import_tests(reference-path-import-diamond "${reference_path_dir}" import_diamond plain_module via_module) tslang_add_import_tests(reference-path-import-cycle "${reference_path_dir}" import_cycle cycle_module_a cycle_module_b) # Through a -shared library of these two modules the importer read Node twice ("redefinition of diff --git a/tslang/test/tester/import-chain/chain_01.ts b/tslang/test/tester/import-chain/chain_01.ts new file mode 100644 index 000000000..79b3bb526 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_01.ts @@ -0,0 +1,3 @@ +// Link 1 of the chain deep_chain.ts imports; see there. + +export function chain1() { return 1; } diff --git a/tslang/test/tester/import-chain/chain_02.ts b/tslang/test/tester/import-chain/chain_02.ts new file mode 100644 index 000000000..f7df70251 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_02.ts @@ -0,0 +1,4 @@ +// Link 2 of the chain deep_chain.ts imports; see there. +import './chain_01' + +export function chain2() { return chain1() + 2; } diff --git a/tslang/test/tester/import-chain/chain_03.ts b/tslang/test/tester/import-chain/chain_03.ts new file mode 100644 index 000000000..b5cef1be5 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_03.ts @@ -0,0 +1,5 @@ +// Link 3 of the chain deep_chain.ts imports; see there. +import './chain_02' +import './chain_01' + +export function chain3() { return chain2() + 3; } diff --git a/tslang/test/tester/import-chain/chain_04.ts b/tslang/test/tester/import-chain/chain_04.ts new file mode 100644 index 000000000..df5719c0e --- /dev/null +++ b/tslang/test/tester/import-chain/chain_04.ts @@ -0,0 +1,5 @@ +// Link 4 of the chain deep_chain.ts imports; see there. +import './chain_03' +import './chain_01' + +export function chain4() { return chain3() + 4; } diff --git a/tslang/test/tester/import-chain/chain_05.ts b/tslang/test/tester/import-chain/chain_05.ts new file mode 100644 index 000000000..a7e9aee94 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_05.ts @@ -0,0 +1,5 @@ +// Link 5 of the chain deep_chain.ts imports; see there. +import './chain_04' +import './chain_01' + +export function chain5() { return chain4() + 5; } diff --git a/tslang/test/tester/import-chain/chain_06.ts b/tslang/test/tester/import-chain/chain_06.ts new file mode 100644 index 000000000..43aeba38d --- /dev/null +++ b/tslang/test/tester/import-chain/chain_06.ts @@ -0,0 +1,5 @@ +// Link 6 of the chain deep_chain.ts imports; see there. +import './chain_05' +import './chain_01' + +export function chain6() { return chain5() + 6; } diff --git a/tslang/test/tester/import-chain/chain_07.ts b/tslang/test/tester/import-chain/chain_07.ts new file mode 100644 index 000000000..c9043099e --- /dev/null +++ b/tslang/test/tester/import-chain/chain_07.ts @@ -0,0 +1,5 @@ +// Link 7 of the chain deep_chain.ts imports; see there. +import './chain_06' +import './chain_01' + +export function chain7() { return chain6() + 7; } diff --git a/tslang/test/tester/import-chain/chain_08.ts b/tslang/test/tester/import-chain/chain_08.ts new file mode 100644 index 000000000..1f25afaaf --- /dev/null +++ b/tslang/test/tester/import-chain/chain_08.ts @@ -0,0 +1,5 @@ +// Link 8 of the chain deep_chain.ts imports; see there. +import './chain_07' +import './chain_01' + +export function chain8() { return chain7() + 8; } diff --git a/tslang/test/tester/import-chain/chain_09.ts b/tslang/test/tester/import-chain/chain_09.ts new file mode 100644 index 000000000..d50083ea7 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_09.ts @@ -0,0 +1,5 @@ +// Link 9 of the chain deep_chain.ts imports; see there. +import './chain_08' +import './chain_01' + +export function chain9() { return chain8() + 9; } diff --git a/tslang/test/tester/import-chain/chain_10.ts b/tslang/test/tester/import-chain/chain_10.ts new file mode 100644 index 000000000..960271b29 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_10.ts @@ -0,0 +1,5 @@ +// Link 10 of the chain deep_chain.ts imports; see there. +import './chain_09' +import './chain_01' + +export function chain10() { return chain9() + 10; } diff --git a/tslang/test/tester/import-chain/chain_11.ts b/tslang/test/tester/import-chain/chain_11.ts new file mode 100644 index 000000000..b256cc36a --- /dev/null +++ b/tslang/test/tester/import-chain/chain_11.ts @@ -0,0 +1,5 @@ +// Link 11 of the chain deep_chain.ts imports; see there. +import './chain_10' +import './chain_01' + +export function chain11() { return chain10() + 11; } diff --git a/tslang/test/tester/import-chain/chain_12.ts b/tslang/test/tester/import-chain/chain_12.ts new file mode 100644 index 000000000..9a2dafb79 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_12.ts @@ -0,0 +1,5 @@ +// Link 12 of the chain deep_chain.ts imports; see there. +import './chain_11' +import './chain_01' + +export function chain12() { return chain11() + 12; } diff --git a/tslang/test/tester/import-chain/chain_13.ts b/tslang/test/tester/import-chain/chain_13.ts new file mode 100644 index 000000000..5cff1a0b6 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_13.ts @@ -0,0 +1,5 @@ +// Link 13 of the chain deep_chain.ts imports; see there. +import './chain_12' +import './chain_01' + +export function chain13() { return chain12() + 13; } diff --git a/tslang/test/tester/import-chain/chain_14.ts b/tslang/test/tester/import-chain/chain_14.ts new file mode 100644 index 000000000..5479d78cc --- /dev/null +++ b/tslang/test/tester/import-chain/chain_14.ts @@ -0,0 +1,5 @@ +// Link 14 of the chain deep_chain.ts imports; see there. +import './chain_13' +import './chain_01' + +export function chain14() { return chain13() + 14; } diff --git a/tslang/test/tester/import-chain/chain_15.ts b/tslang/test/tester/import-chain/chain_15.ts new file mode 100644 index 000000000..3f242a260 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_15.ts @@ -0,0 +1,5 @@ +// Link 15 of the chain deep_chain.ts imports; see there. +import './chain_14' +import './chain_01' + +export function chain15() { return chain14() + 15; } diff --git a/tslang/test/tester/import-chain/chain_16.ts b/tslang/test/tester/import-chain/chain_16.ts new file mode 100644 index 000000000..838968afe --- /dev/null +++ b/tslang/test/tester/import-chain/chain_16.ts @@ -0,0 +1,5 @@ +// Link 16 of the chain deep_chain.ts imports; see there. +import './chain_15' +import './chain_01' + +export function chain16() { return chain15() + 16; } diff --git a/tslang/test/tester/import-chain/chain_17.ts b/tslang/test/tester/import-chain/chain_17.ts new file mode 100644 index 000000000..fc38d324b --- /dev/null +++ b/tslang/test/tester/import-chain/chain_17.ts @@ -0,0 +1,5 @@ +// Link 17 of the chain deep_chain.ts imports; see there. +import './chain_16' +import './chain_01' + +export function chain17() { return chain16() + 17; } diff --git a/tslang/test/tester/import-chain/chain_18.ts b/tslang/test/tester/import-chain/chain_18.ts new file mode 100644 index 000000000..76f5827b1 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_18.ts @@ -0,0 +1,5 @@ +// Link 18 of the chain deep_chain.ts imports; see there. +import './chain_17' +import './chain_01' + +export function chain18() { return chain17() + 18; } diff --git a/tslang/test/tester/import-chain/chain_19.ts b/tslang/test/tester/import-chain/chain_19.ts new file mode 100644 index 000000000..008fdbb38 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_19.ts @@ -0,0 +1,5 @@ +// Link 19 of the chain deep_chain.ts imports; see there. +import './chain_18' +import './chain_01' + +export function chain19() { return chain18() + 19; } diff --git a/tslang/test/tester/import-chain/chain_20.ts b/tslang/test/tester/import-chain/chain_20.ts new file mode 100644 index 000000000..ac748bfa2 --- /dev/null +++ b/tslang/test/tester/import-chain/chain_20.ts @@ -0,0 +1,5 @@ +// Link 20 of the chain deep_chain.ts imports; see there. +import './chain_19' +import './chain_01' + +export function chain20() { return chain19() + 20; } diff --git a/tslang/test/tester/import-chain/deep_chain.ts b/tslang/test/tester/import-chain/deep_chain.ts new file mode 100644 index 000000000..2f71d8c97 --- /dev/null +++ b/tslang/test/tester/import-chain/deep_chain.ts @@ -0,0 +1,8 @@ +// A chain of 20 imports, each module importing the one below it and chain_01 as well. Each +// import was discovered again in every module it was generated into, so a module nested d +// deep was generated 2^d times: this one took minutes and gigabytes, and the TIMEOUT fails it. +import './chain_20' + +assert(chain20() == 210); + +print("done.");