Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions tslang/lib/TypeScript/MLIRGenImpl.h
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,23 @@ class MLIRGenImpl

mlir::LogicalResult mlirGenImportSharedLib(mlir::Location location, StringRef filePath, bool dynamic, const GenContext &genContext);

// A library an `import` names: its file, and whether it is loaded at run time (no @static).
struct ImportedLibrary
{
std::string path;
bool dynamic;
};

// The library file an `import` of `specifier` points to when that file exists, named as the
// target names libraries.
std::string importLibraryPath(StringRef specifier);

// A library that the imports of the source module `fullPath` reach - its own, or those of the
// source modules it imports, and so on - and that holds the module itself (exports one of
// `declSymbols`).
std::optional<ImportedLibrary> libraryHoldingModule(mlir::Location location, StringRef fullPath,
ArrayRef<std::string> declSymbols);

void addImportedOwnNoDrops(StringRef factsText);

void addImportedOwnObjectFacts(StringRef factsText);
Expand Down Expand Up @@ -16814,6 +16831,23 @@ class MLIRGenImpl
// imports a) stops instead of recursing until the stack overflows.
llvm::StringSet<> filesInProgress;

// The declaration symbols (__decls_<moduleSymbolSuffix>, and the _generic_ one) of the files
// in filesInProgress. A library imported while one of them is generated must not declare that
// one: its source is declaring it right now, and the two redefine each other.
llvm::StringSet<> declSymbolsInProgress;

// What a source module's top-level imports name, read from its text once
// (libraryHoldingModule): the source modules, by canonicalFilePath, and the libraries.
struct ModuleImports
{
std::vector<std::string> sources;
std::vector<ImportedLibrary> libraries;
};
llvm::StringMap<ModuleImports> moduleImportsCache;

// The declaration symbols each library libraryHoldingModule looked into exports.
llvm::StringMap<llvm::StringSet<>> libraryDeclSymbolsCache;

// Source imports whose own discovery pass has run once (mlirGenInclude). Their declarations
// are in the maps from then on, so an import generated into another module - each discovery
// module, then theModule - is generated without discovering it again. Discovered again each
Expand Down
169 changes: 150 additions & 19 deletions tslang/lib/TypeScript/MLIRGenModule.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,10 @@ namespace mlirgen

// an imported module importing the program back does not generate the program again
filesInProgress.insert(fileKey(module));
// nor does a library holding the program declare the program again
auto programSuffix = moduleSymbolSuffix(fileKey(module));
declSymbolsInProgress.insert(std::string(SHARED_LIB_DECLARATIONS_2UNDERSCORE "_") + programSuffix);
declSymbolsInProgress.insert(std::string(SHARED_LIB_DECLARATIONS_2UNDERSCORE "_generic_") + programSuffix);

stage = Stages::Discovering;
auto storeDebugInfo = compileOptions.generateDebugInfo;
Expand Down Expand Up @@ -1367,8 +1371,25 @@ namespace mlirgen
return mlir::success();
}

// Held by a library that its own imports reach: test-runner -shared links every module
// but the program into one, so with main -> m2 -> m1 the library m1 holds m2 as well. The
// program finds the module there - linked with the library alone, it has no other
// definition of what the module's source declares - so the library is imported in its
// place. Found before the source is read: once generated, even by discovery, the source's
// declarations would stand beside the library's.
if (auto library = libraryHoldingModule(location, fullPath, {declSymbol, genericDeclSymbol}))
{
return mlirGenImportSharedLib(location, library->path, library->dynamic, genContext);
}

filesInProgress.insert(canonicalPath);
auto inProgress = llvm::make_scope_exit([&]() { filesInProgress.erase(canonicalPath); });
declSymbolsInProgress.insert(declSymbol);
declSymbolsInProgress.insert(genericDeclSymbol);
auto inProgress = llvm::make_scope_exit([&]() {
filesInProgress.erase(canonicalPath);
declSymbolsInProgress.erase(declSymbol);
declSymbolsInProgress.erase(genericDeclSymbol);
});

// An import that points to a .ts file: compiled, it is a declaration - the module is
// compiled into an object of its own, and the objects are linked into one program. The
Expand Down Expand Up @@ -1471,6 +1492,13 @@ namespace mlirgen
return mlir::success();
}

// Read the same wherever the import is: inside a module imported as source, which is
// generated in declaration mode (mlirGenInclude), the globals through which this module
// calls the library's functions became external, and their initializers - the symbol
// lookups - were left at the module's top, outside any function.
MLIRValueGuard<bool> declarationModeGuard(declarationMode);
declarationMode = false;

// A library built for another architecture or OS (an i686 DLL for this x64 compiler)
// cannot be loaded into this process. Its declarations are then read from the file; the
// program still loads it at run time, through the global constructor below.
Expand Down Expand Up @@ -1663,11 +1691,13 @@ namespace mlirgen

// A library can hold several modules (test-runner -shared links them all into one), each
// with its own __decls_<module>. A module already in this module - imported as source,
// or through another library - is not declared again; see mlirGenInclude.
// or through another library - is not declared again; see mlirGenInclude. Nor is one
// whose source is being generated around this import (declSymbolsInProgress): the
// program, say, built into a library it imports. Its source declares it.
SmallVector<StringRef> declaredSymbols;
for (auto declSymbol : symbols)
{
if (emittedFiles.contains(declSymbol))
if (emittedFiles.contains(declSymbol) || declSymbolsInProgress.contains(declSymbol))
{
continue;
}
Expand Down Expand Up @@ -1784,31 +1814,19 @@ namespace mlirgen
return mlir::success();
}

mlir::LogicalResult MLIRGenImpl::mlirGen(ImportDeclaration importDeclarationAST, const GenContext &genContext)
std::string MLIRGenImpl::importLibraryPath(StringRef specifier)
{
auto location = loc(importDeclarationAST);

auto result = mlirGen(importDeclarationAST->moduleSpecifier, genContext);
EXIT_IF_FAILED_OR_NO_VALUE(result)
auto modulePath = V(result);

auto constantOp = modulePath.getDefiningOp<mlir_ts::ConstantOp>();
assert(constantOp);
auto valueAttr = mlir::cast<mlir::StringAttr>(constantOp.getValueAttr());

auto stringVal = valueAttr.getValue();

// The library is named as the target names it, as --emit=dll does (utils.cpp): the path is
// also what the program loads at run time, so a host's naming would not be found there.
auto targetIsWindows = llvm::Triple(compileOptions.moduleTargetTriple).getOS() == llvm::Triple::Win32;

std::string fullPath;
fullPath += stringVal;
fullPath += specifier;
if (!targetIsWindows)
{
// rebuild file path
auto fileName = sys::path::filename(stringVal);
auto path = stringVal.substr(0, stringVal.size() - fileName.size());
auto fileName = sys::path::filename(specifier);
auto path = specifier.substr(0, specifier.size() - fileName.size());
fullPath = path;
fullPath += "lib";
fullPath += fileName;
Expand All @@ -1819,6 +1837,119 @@ namespace mlirgen
fullPath += targetIsWindows ? ".dll" : ".so";
}

return fullPath;
}

std::optional<MLIRGenImpl::ImportedLibrary> MLIRGenImpl::libraryHoldingModule(mlir::Location location, StringRef fullPath,
ArrayRef<std::string> declSymbols)
{
#ifdef GENERATE_IMPORT_INFO_USING_D_TS_FILE
// a library's declarations are not told apart by module there
return std::nullopt;
#else
auto libraryHolds = [&](StringRef libraryPath) {
auto cached = libraryDeclSymbolsCache.find(libraryPath);
if (cached == libraryDeclSymbolsCache.end())
{
SmallVector<StringRef> symbolsAll;
Dump::getSymbols(libraryPath, symbolsAll, stringAllocator);
llvm::StringSet<> declSymbolsOfLibrary;
for (auto symbol : symbolsAll)
{
if (symbol.starts_with(SHARED_LIB_DECLARATIONS_2UNDERSCORE))
{
declSymbolsOfLibrary.insert(symbol);
}
}

cached = libraryDeclSymbolsCache.insert({libraryPath, std::move(declSymbolsOfLibrary)}).first;
}

return llvm::any_of(declSymbols, [&](const std::string &symbol) { return cached->second.contains(symbol); });
};

// every source module reached through imports, each read once (moduleImportsCache)
llvm::StringSet<> visited;
SmallVector<std::string> pending{canonicalFilePath(fullPath)};
while (!pending.empty())
{
auto file = pending.pop_back_val();
if (!visited.insert(file).second)
{
continue;
}

auto cached = moduleImportsCache.find(file);
if (cached == moduleImportsCache.end())
{
ModuleImports imports;
// a file that does not open is the import's to report
if (sys::fs::exists(file) && !StringRef(file).ends_with(".d.ts"))
{
auto [source, includeFiles] = loadIncludeFile(location, file);
if (source)
{
for (auto statement : source->statements)
{
if (statement != SyntaxKind::ImportDeclaration)
{
continue;
}

auto importDeclaration = statement.as<ImportDeclaration>();
if (importDeclaration->moduleSpecifier != SyntaxKind::StringLiteral)
{
continue;
}

auto specifier = convertWideToUTF8(importDeclaration->moduleSpecifier.as<ts::StringLiteral>()->text);
auto libraryPath = importLibraryPath(specifier);
if (sys::fs::exists(libraryPath))
{
imports.libraries.push_back({libraryPath, !MLIRHelper::hasDecorator(importDeclaration, "static")});
}
else
{
imports.sources.push_back(canonicalFilePath(includeFilePath(specifier)));
}
}
}
}

cached = moduleImportsCache.insert({file, std::move(imports)}).first;
}

auto &imports = cached->second;
for (auto &library : imports.libraries)
{
if (libraryHolds(library.path))
{
return library;
}
}

pending.append(imports.sources.begin(), imports.sources.end());
}

return std::nullopt;
#endif
}

mlir::LogicalResult MLIRGenImpl::mlirGen(ImportDeclaration importDeclarationAST, const GenContext &genContext)
{
auto location = loc(importDeclarationAST);

auto result = mlirGen(importDeclarationAST->moduleSpecifier, genContext);
EXIT_IF_FAILED_OR_NO_VALUE(result)
auto modulePath = V(result);

auto constantOp = modulePath.getDefiningOp<mlir_ts::ConstantOp>();
assert(constantOp);
auto valueAttr = mlir::cast<mlir::StringAttr>(constantOp.getValueAttr());

auto stringVal = valueAttr.getValue();

auto fullPath = importLibraryPath(stringVal);
if (sys::fs::exists(fullPath))
{
//auto dynamic = MLIRHelper::hasDecorator(importDeclarationAST, "dynamic");
Expand Down
14 changes: 11 additions & 3 deletions tslang/test/tester/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5154,14 +5154,22 @@ foreach(import_chain_index RANGE 1 20)
endif()
list(APPEND import_chain_modules "${PROJECT_SOURCE_DIR}/test/tester/import-chain/chain_${import_chain_index}.ts")
endforeach()
# Not -shared: a module that imports another is declared twice in the one library, chain or not.
# The JIT compiles each import into its cache by itself.
# The JIT compiles each import into its cache by itself. Under -shared the library is chain_01,
# reached only at the bottom of the chain, and holds every link: chain_20 is imported from it.
tslang_add_test(NAME test-compile-import-chain-deep
COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/import-chain/deep_chain.ts" ${import_chain_modules})
tslang_add_test(NAME test-jit-import-chain-deep
COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/import-chain/deep_chain.ts")
set_tests_properties(test-compile-import-chain-deep test-jit-import-chain-deep PROPERTIES TIMEOUT 120)
tslang_add_test(NAME test-compile-shared-import-chain-deep
COMMAND test-runner -shared "${PROJECT_SOURCE_DIR}/test/tester/import-chain/deep_chain.ts" ${import_chain_modules})
tslang_add_test(NAME test-jit-shared-import-chain-deep
COMMAND test-runner -jit -shared "${PROJECT_SOURCE_DIR}/test/tester/import-chain/deep_chain.ts" ${import_chain_modules})
set_tests_properties(test-compile-import-chain-deep test-jit-import-chain-deep test-compile-shared-import-chain-deep
test-jit-shared-import-chain-deep PROPERTIES TIMEOUT 120)
tslang_add_import_tests(reference-path-import-diamond "${reference_path_dir}" import_diamond plain_module via_module)
# The library is named after nested_bottom, the first module, and holds nested_middle as well: the
# program imports nested_middle as source and meets the library only inside it.
tslang_add_import_tests(reference-path-import-nested "${reference_path_dir}" nested_import nested_bottom nested_middle)
tslang_add_import_tests(reference-path-import-cycle "${reference_path_dir}" import_cycle cycle_module_a cycle_module_b)
# Through a -shared library of these two modules the importer read Node twice ("redefinition of
# symbol named 'Node..new'"), and then called its methods through the wrong vtable slots - see
Expand Down
2 changes: 2 additions & 0 deletions tslang/test/tester/reference-path/nested_bottom.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
// The bottom of nested_import.ts's chain; see there.
export function bottom_fn() { return 1; }
9 changes: 9 additions & 0 deletions tslang/test/tester/reference-path/nested_import.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
// The program imports nested_middle, which imports nested_bottom. Under test-runner -shared the
// library is named after nested_bottom and holds nested_middle too, which the program only learns
// inside nested_middle: it was declared both from its source and from the library ("redefinition
// of symbol named 'nested_fn'"). The library is imported in its place.
import './nested_middle'

assert(nested_fn() == 3);

print("done.");
4 changes: 4 additions & 0 deletions tslang/test/tester/reference-path/nested_middle.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// Imported by nested_import.ts; imports nested_bottom.ts.
import './nested_bottom'

export function nested_fn() { return bottom_fn() + 2; }
Loading