Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 40 additions & 1 deletion tslang/lib/TypeScript/MLIRGenImpl.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/Path.h"
Expand Down Expand Up @@ -223,6 +224,24 @@ class MLIRGenImpl
std::pair<SourceFile, std::vector<SourceFile>> loadImportFile(mlir::Location location, StringRef fullPath,
StringRef canonicalPath);

// One file's entry in importUnits.
struct ImportUnit
{
std::vector<std::pair<StringRef, VariableDeclarationDOM::TypePtr>> globals;
std::vector<std::string> children;
bool recorded = false;
bool skippable = true;
};

// importUnits: the file being recorded (top of importRecordStack), if any.
ImportUnit *recordingImportUnit();
// Generates `includeFile`, a file a source import references, recording it (importUnits).
mlir::LogicalResult mlirGenRecordedInclude(SourceFile includeFile, const GenContext &genContext);
// Puts back into this discovery module's fullNameGlobalsMap the globals generating `path`
// registered, and those of every file it reached. False when one of them was not recorded or
// is not skippable: the import is then generated.
bool replayImportGlobals(StringRef path);

// The one name of a file, however it was spelled: `./x.d.ts`, `sub/../x.d.ts` and `x.d.ts`
// are the same file. Keys `emittedFiles` and `filesInProgress`.
static std::string canonicalFilePath(StringRef filePath);
Expand Down Expand Up @@ -14742,7 +14761,8 @@ class MLIRGenImpl
// `emittedFiles`: discovery has to see every declaration to resolve against it.
DiscoveryModuleScope(MLIRGenImpl &mlirGenImpl)
: moduleGuard(mlirGenImpl.theModule), insertGuard(mlirGenImpl.builder),
emittedFilesGuard(mlirGenImpl.emittedFiles, {})
emittedFilesGuard(mlirGenImpl.emittedFiles, {}),
discoveryDepthGuard(mlirGenImpl.discoveryDepth, mlirGenImpl.discoveryDepth + 1)
{
discoveryModule =
mlir::ModuleOp::create(mlirGenImpl.theModule.getLoc(), mlir::StringRef("discovery_module"));
Expand All @@ -14760,6 +14780,7 @@ class MLIRGenImpl
MLIRValueGuard<mlir::ModuleOp> moduleGuard;
mlir::OpBuilder::InsertionGuard insertGuard;
MLIRValueGuard<llvm::StringSet<>> emittedFilesGuard;
MLIRValueGuard<int> discoveryDepthGuard;
mlir::ModuleOp discoveryModule;
};

Expand Down Expand Up @@ -16866,6 +16887,24 @@ class MLIRGenImpl
// its compile.
llvm::StringMap<std::pair<SourceFile, std::vector<SourceFile>>> parsedImports;

// How many discovery modules (DiscoveryModuleScope) enclose the current generation: above 0,
// theModule is a throwaway one. GenContext::dummyRun does not say it: a module generated
// inside discovery (mlirCodeGenModule) starts a GenContext of its own.
int discoveryDepth = 0;

// What generating a file (a source import, or a file one references) registered that a
// discovery module needs from it when the import is not generated there again
// (replayImportGlobals): the globals it put in fullNameGlobalsMap - scoped to each module,
// the class's .rtti among them - and the files it reached, whose globals come with it.
// Recorded the first time the file is generated (importRecordStack). A file that reached a
// library, or an import cycle, is not `skippable`: it is generated each time, as before.
llvm::StringMap<ImportUnit> importUnits;
// The file being generated, innermost last; "" for one whose first generation is recorded
// already, or for the program.
std::vector<std::string> importRecordStack;
llvm::BumpPtrAllocator importKeyAllocator;
llvm::StringSaver importKeySaver{importKeyAllocator};

// The default library's files - lib.d.ts and every file it references - by file key. Its
// ambient data is defined in its DLL when the program links that (see
// isImportedFromSharedDefaultLib).
Expand Down
211 changes: 207 additions & 4 deletions tslang/lib/TypeScript/MLIRGenModule.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,138 @@ namespace mlirgen
return cached->second;
}

MLIRGenImpl::ImportUnit *MLIRGenImpl::recordingImportUnit()
{
if (importRecordStack.empty() || importRecordStack.back().empty())
{
return nullptr;
}

return &importUnits[importRecordStack.back()];
}

mlir::LogicalResult MLIRGenImpl::mlirGenRecordedInclude(SourceFile includeFile, const GenContext &genContext)
{
auto key = fileKey(includeFile);
importRecordStack.push_back(importUnits[key].recorded ? std::string() : key);
auto result = mlirGen(includeFile->statements, genContext);
auto recorded = importRecordStack.back();
importRecordStack.pop_back();
if (!recorded.empty())
{
if (mlir::succeeded(result))
{
importUnits[recorded].recorded = true;
}
else
{
importUnits.erase(recorded);
}
}

return result;
}

bool MLIRGenImpl::replayImportGlobals(StringRef path)
{
llvm::StringSet<> visited;
mlir::SmallVector<ImportUnit *> units;

// A file `importKey` references, generated into this module the first time it is reached
// here, as generating the import would (mlirCodeGenModule); recorded on the way. A file a
// module referenced is emitted by its discovery's dummy run first, so a nested import
// finds it there and skips it: it is first generated, and recorded, only here.
auto generateReferencedFile = [&](StringRef importKey, StringRef key) {
auto parsed = parsedImports.find(importKey);
if (parsed == parsedImports.end())
{
return false;
}

for (auto includeFile : parsed->second.second)
{
if (fileKey(includeFile) != key)
{
continue;
}

mlir::SmallVector<std::unique_ptr<mlir::Diagnostic>> postponedMessages;
mlir::ScopedDiagnosticHandler diagHandler(builder.getContext(), [&](mlir::Diagnostic &diag) {
postponedMessages.emplace_back(new mlir::Diagnostic(std::move(diag)));
});

GenContext genContext{};
genContext.rootContext = &genContext;
genContext.postponedMessages = &postponedMessages;

SourceFileScope sourceFileScope(*this, includeFile);
if (mlir::failed(mlirGenRecordedInclude(includeFile, genContext)))
{
return false;
}

emittedFiles.insert(key);
return true;
}

return false;
};

std::function<bool(StringRef, StringRef)> collect = [&](StringRef parentKey, StringRef key) {
if (!visited.insert(key).second)
{
return true;
}

// in this module already, and so are its globals
if (emittedFiles.contains(key))
{
return true;
}

auto found = importUnits.find(key);
if ((found == importUnits.end() || !found->second.recorded) && !parentKey.empty() &&
generateReferencedFile(parentKey, key))
{
return true;
}

if (found == importUnits.end() || !found->second.recorded || !found->second.skippable)
{
return false;
}

units.push_back(&found->second);
for (auto &child : found->second.children)
{
if (!collect(key, child))
{
return false;
}
}

return true;
};

if (!collect(StringRef(), path))
{
return false;
}

for (auto *unit : units)
{
for (auto &[name, variableDeclaration] : unit->globals)
{
if (!fullNameGlobalsMap.count(name))
{
fullNameGlobalsMap.insert(name, variableDeclaration);
}
}
}

return true;
}

mlir::LogicalResult MLIRGenImpl::showMessages(SourceFile module, std::vector<SourceFile> includeFiles)
{
mlir::ScopedDiagnosticHandler diagHandler(builder.getContext(), [&](mlir::Diagnostic &diag) {
Expand Down Expand Up @@ -1141,6 +1273,10 @@ namespace mlirgen
// (e.g. default-lib function bodies such as 'console.log') that must survive.
DiscoveryModuleScope discoveryModuleScope(*this);

// what a dummy run registers is not what generating the file registers (importUnits)
importRecordStack.push_back(std::string());
auto notRecording = llvm::make_scope_exit([&]() { importRecordStack.pop_back(); });

// Process of discovery here
GenContext genContextPartial{};
genContextPartial.allowPartialResolve = true;
Expand Down Expand Up @@ -1211,14 +1347,19 @@ namespace mlirgen
// a file the program, or another import, already emitted into this module is skipped
for (auto includeFile : includeFiles)
{
if (auto *unit = recordingImportUnit())
{
unit->children.push_back(fileKey(includeFile));
}

if (emittedFiles.contains(fileKey(includeFile)))
{
continue;
}

SourceFileScope sourceFileScope(*this, includeFile);

if (failed(mlirGen(includeFile->statements, genContext)))
if (failed(mlirGenRecordedInclude(includeFile, genContext)))
{
outputDiagnostics(postponedMessages, 1);
return mlir::failure();
Expand Down Expand Up @@ -1391,6 +1532,13 @@ namespace mlirgen
auto fullPath = includeFilePath(filePath);
auto canonicalPath = canonicalFilePath(fullPath);

// the file being recorded reached this one: its globals come with it (replayImportGlobals)
auto *recordingUnit = recordingImportUnit();
if (recordingUnit)
{
recordingUnit->children.push_back(canonicalPath);
}

// Already in this module - referenced with `/// <reference path>`, or imported directly
// and through another module - so its declarations are all there. Or an import cycle back
// to a file still being generated: generating it again would never end; what the cycle
Expand All @@ -1403,6 +1551,11 @@ namespace mlirgen
if (filesInProgress.contains(canonicalPath))
{
compileOptions.importCycle = true;
if (recordingUnit)
{
recordingUnit->skippable = false;
}

return mlir::success();
}

Expand All @@ -1413,6 +1566,11 @@ namespace mlirgen
auto genericDeclSymbol = std::string(SHARED_LIB_DECLARATIONS_2UNDERSCORE "_generic_") + suffix;
if (emittedFiles.contains(declSymbol) || emittedFiles.contains(genericDeclSymbol))
{
if (recordingUnit)
{
recordingUnit->skippable = false;
}

return mlir::success();
}

Expand All @@ -1424,9 +1582,24 @@ namespace mlirgen
// declarations would stand beside the library's.
if (auto library = libraryHoldingModule(location, fullPath, {declSymbol, genericDeclSymbol}))
{
if (recordingUnit)
{
recordingUnit->skippable = false;
}

return mlirGenImportSharedLib(location, library->path, library->dynamic, genContext);
}

// In a discovery module, an import discovered once already is not generated again: what
// discovery resolves against it is in the compile's maps, but for the globals it put in
// fullNameGlobalsMap, which belongs to each module and is put back. theModule still
// generates every import. Generated again into each discovery module that reached it, an
// import was generated 211 times for powder_sandbox's checks.ts.
if (discoveryDepth > 0 && discoveredImports.contains(canonicalPath) && replayImportGlobals(canonicalPath))
{
return mlir::success();
}

filesInProgress.insert(canonicalPath);
declSymbolsInProgress.insert(declSymbol);
declSymbolsInProgress.insert(genericDeclSymbol);
Expand Down Expand Up @@ -1491,9 +1664,32 @@ namespace mlirgen

// discovered once already (generated into an earlier module): its declarations are
// known, and discovering it again would generate every import inside it twice more
generated = (discoveredImports.contains(canonicalPath) ||
mlir::succeeded(mlirDiscoverAllDependencies(importSource, importIncludeFiles))) &&
mlir::succeeded(mlirCodeGenModule(importSource, importIncludeFiles, false, false));
generated = discoveredImports.contains(canonicalPath) ||
mlir::succeeded(mlirDiscoverAllDependencies(importSource, importIncludeFiles));
if (generated)
{
// the first generation that succeeds is recorded (importUnits)
auto record = !importUnits[canonicalPath].recorded;
if (record)
{
importUnits.erase(canonicalPath);
}

importRecordStack.push_back(record ? canonicalPath : std::string());
generated = mlir::succeeded(mlirCodeGenModule(importSource, importIncludeFiles, false, false));
importRecordStack.pop_back();
if (record)
{
if (generated)
{
importUnits[canonicalPath].recorded = true;
}
else
{
importUnits.erase(canonicalPath);
}
}
}
}

// the import's own diagnostic handlers are gone: these reach the importer's
Expand Down Expand Up @@ -1532,6 +1728,13 @@ namespace mlirgen
// Imported into this module already - directly and through another module, say: it is
// loaded, and its declarations are here. A second copy of them redefines every symbol.
auto canonicalPath = canonicalFilePath(filePath);

// what a library declares is not recorded: a file that imports one is generated each time
if (auto *unit = recordingImportUnit())
{
unit->skippable = false;
}

if (emittedFiles.contains(canonicalPath))
{
return mlir::success();
Expand Down
4 changes: 4 additions & 0 deletions tslang/lib/TypeScript/MLIRGenVariables.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,10 @@ namespace mlirgen
else if (variableDeclarationInfo.isFullName)
{
fullNameGlobalsMap.insert(variableDeclarationInfo.fullName, variableDeclaration);
if (auto *unit = recordingImportUnit())
{
unit->globals.push_back({importKeySaver.save(variableDeclarationInfo.fullName), variableDeclaration});
}
}
else
{
Expand Down
6 changes: 6 additions & 0 deletions tslang/test/tester/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5182,6 +5182,12 @@ tslang_add_test(NAME test-jit-shared-import-chain-deep
COMMAND test-runner -jit -shared "${PROJECT_SOURCE_DIR}/test/tester/import-chain/deep_chain.ts" ${import_chain_modules})
set_tests_properties(test-compile-import-chain-deep test-jit-import-chain-deep test-compile-shared-import-chain-deep
test-jit-shared-import-chain-deep PROPERTIES TIMEOUT 120)
# An import discovered already is not generated again into a discovery module; the class .rtti
# global two imports down has to be put back for the program's discovery to read it.
tslang_add_import_tests(import-chain-rtti-skipped "${PROJECT_SOURCE_DIR}/test/tester/import-chain" rtti_skipped_import
rtti_bottom rtti_middle rtti_top)
tslang_add_test(NAME test-jit-import-chain-rtti-skipped
COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/import-chain/rtti_skipped_import.ts")
tslang_add_import_tests(reference-path-import-diamond "${reference_path_dir}" import_diamond plain_module via_module)
# The library is named after nested_bottom, the first module, and holds nested_middle as well: the
# program imports nested_middle as source and meets the library only inside it.
Expand Down
10 changes: 10 additions & 0 deletions tslang/test/tester/import-chain/rtti_bottom.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// The class whose .rtti the program reads through two imports; see rtti_skipped_import.ts.

export class Profiler {
reads = 0;

read(): boolean {
this.reads++;
return this.reads > 1;
}
}
Loading
Loading