From c2944379b39451c539b866ff25c51d992ddf9b98 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:39:21 -0700 Subject: [PATCH 1/6] compiler: make equal-position member ordering deterministic Generated CGo globals can share a source position. Break these ties by name so independent compilations produce the same bitcode and ThinLTO symbol names instead of depending on Go map iteration order. --- compiler/compiler.go | 2 +- compiler/compiler_test.go | 19 +++++++++++++++++++ compiler/testdata/cgo-global-order.go | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 compiler/testdata/cgo-global-order.go diff --git a/compiler/compiler.go b/compiler/compiler.go index e4bb613840..84aabd5dee 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -883,7 +883,7 @@ func (c *compilerContext) createPackage(irbuilder llvm.Builder, pkg *ssa.Package sort.Slice(members, func(i, j int) bool { iPos := pkg.Members[members[i]].Pos() jPos := pkg.Members[members[j]].Pos() - if i == j { + if iPos == jPos { // Cannot sort by pos, so do it by name. return members[i] < members[j] } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index 4831a6868e..d66b26b616 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -137,6 +137,25 @@ func TestCompiler(t *testing.T) { } } +func TestCGoGlobalOrder(t *testing.T) { + for range 10 { + mod, errs := testCompilePackage(t, &compileopts.Options{Target: "wasm"}, "cgo-global-order.go") + if len(errs) != 0 { + t.Fatal(errs) + } + var names []string + for global := mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if strings.HasSuffix(global.Name(), "$funcaddr") { + names = append(names, global.Name()) + } + } + mod.Dispose() + if len(names) != 4 || !slices.IsSorted(names) { + t.Fatalf("CGo globals are not in name order: %v", names) + } + } +} + func TestOptimizedLargeAggregateABI(t *testing.T) { options := &compileopts.Options{Target: "wasm"} mod, errs := testCompilePackage(t, options, "large-optimized.go") diff --git a/compiler/testdata/cgo-global-order.go b/compiler/testdata/cgo-global-order.go new file mode 100644 index 0000000000..b72251be45 --- /dev/null +++ b/compiler/testdata/cgo-global-order.go @@ -0,0 +1,16 @@ +package main + +/* +void d(void); +void c(void); +void b(void); +void a(void); +*/ +import "C" + +func main() { + C.d() + C.c() + C.b() + C.a() +} From 9fe95ba6304e8b5a1fd2663402c17756f5defef3 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:04:36 -0700 Subject: [PATCH 2/6] builder: add -trimpath support Local source and cache paths make binaries depend on the build directory and expose the builder filesystem layout. Trim these paths from debug information and C __FILE__ strings, including with -no-debug. Leave the compiled-in runtime.GOROOT empty in trimmed builds, matching Go. Match Go's C compilation directories in both modes so trimpath does not change header lookup. Relative C flags now resolve from the source directory instead of the command's working directory. Keep source paths usable for debugger substitution and package-size reports. Compare independent builds with separate caches so cache reuse does not hide nondeterministic output. Fixes 5275 --- builder/build.go | 104 +++++-- builder/cc.go | 46 ++- builder/library.go | 10 + builder/sizes.go | 12 + builder/sizes_test.go | 51 +++- compileopts/cflags.go | 44 +++ compileopts/config.go | 31 ++ compileopts/options.go | 1 + compiler/compiler.go | 4 +- loader/loader.go | 173 ++++++++++- loader/loader_test.go | 131 +++++++++ main.go | 2 + main_test.go | 268 ++++++++++++++++++ make/config.mk | 2 +- testdata/trimpath/data.txt | 1 + testdata/trimpath/dependency/go.mod | 3 + testdata/trimpath/dependency/include/shared.h | 7 + .../dependency/subpackage/dependency.c | 12 + .../dependency/subpackage/dependency.go | 16 ++ .../dependency/subpackage/include/relative.h | 1 + testdata/trimpath/go.mod | 7 + testdata/trimpath/include/relative.h | 1 + testdata/trimpath/main.c | 5 + testdata/trimpath/main.go | 22 ++ transform/stacksize.go | 6 +- 25 files changed, 920 insertions(+), 40 deletions(-) create mode 100644 compileopts/cflags.go create mode 100644 loader/loader_test.go create mode 100644 testdata/trimpath/data.txt create mode 100644 testdata/trimpath/dependency/go.mod create mode 100644 testdata/trimpath/dependency/include/shared.h create mode 100644 testdata/trimpath/dependency/subpackage/dependency.c create mode 100644 testdata/trimpath/dependency/subpackage/dependency.go create mode 100644 testdata/trimpath/dependency/subpackage/include/relative.h create mode 100644 testdata/trimpath/go.mod create mode 100644 testdata/trimpath/include/relative.h create mode 100644 testdata/trimpath/main.c create mode 100644 testdata/trimpath/main.go diff --git a/builder/build.go b/builder/build.go index dab5a590c2..35cc210dd7 100644 --- a/builder/build.go +++ b/builder/build.go @@ -12,8 +12,10 @@ import ( "encoding/json" "errors" "fmt" + "go/token" "go/types" "hash/crc32" + "io" "maps" "math/bits" "os" @@ -37,7 +39,6 @@ import ( "github.com/tinygo-org/tinygo/transform" "golang.org/x/mod/module" "golang.org/x/mod/semver" - "golang.org/x/tools/go/ssa" "tinygo.org/x/go-llvm" ) @@ -132,6 +133,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe }, "testing": {}, } + if config.TrimPath() { + globalValues["runtime"]["goroot"] = "" + } if config.TestConfig.CompileTestBinary { // The testing.testBinary is set to "1" when in a test. // This is needed for testing.Testing() to work correctly. @@ -213,6 +217,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe RelocationModel: config.RelocationModel(), SizeLevel: sizeLevel, TinyGoVersion: goenv.Version(), + TrimPath: config.TrimPath(), Scheduler: config.Scheduler(), AutomaticStackSize: config.AutomaticStackSize(), @@ -279,6 +284,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe result.PackagePathMap = make(map[string]string, len(lprogram.Packages)) for _, pkg := range lprogram.Sorted() { result.PackagePathMap[pkg.OriginalDir()] = pkg.Pkg.Path() + if config.TrimPath() { + result.PackagePathMap[filepath.FromSlash(pkg.RecordedDir())] = pkg.Pkg.Path() + } } // Strip default initializers for -X globals from the type info before @@ -349,7 +357,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe } } - job.result, err = createEmbedObjectFile(string(data), hexSum, name, pkg.OriginalDir(), tmpdir, compilerConfig) + job.result, err = createEmbedObjectFile(string(data), hexSum, name, pkg.RecordedDir(), tmpdir, compilerConfig) return err }, } @@ -383,7 +391,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe CompilerBuildID: string(compilerBuildID), LLVMVersion: llvm.Version, Config: compilerConfig, - CFlags: pkg.CFlags, + CFlags: pkg.RecordedCFlags(), FileHashes: make(map[string]string, len(pkg.FileHashes)), EmbeddedFiles: make(map[string]string, len(allFiles)), Imports: make(map[string]string, len(pkg.Pkg.Imports())), @@ -391,7 +399,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe UndefinedGlobals: undefinedGlobals, } for filePath, hash := range pkg.FileHashes { - actionID.FileHashes[filePath] = hex.EncodeToString(hash) + actionID.FileHashes[pkg.RecordedPath(filePath)] = hex.EncodeToString(hash) } for name, files := range allFiles { actionID.EmbeddedFiles[name] = files[0].Hash @@ -450,9 +458,10 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // These headers could be compiled in parallel but the benefit // is so small that it's probably not worth parallelizing. // Packages are compiled independently anyway. - for _, cgoHeader := range pkg.CGoHeaders { + packageNameHash := sha256.Sum256([]byte(pkg.ImportPath)) + for i, cgoHeader := range pkg.CGoHeaders { // Store the header text in a temporary file. - f, err := os.CreateTemp(tmpdir, "cgosnippet-*.c") + f, err := os.Create(filepath.Join(tmpdir, fmt.Sprintf("cgosnippet-%x-%d.c", packageNameHash, i))) if err != nil { return err } @@ -464,6 +473,12 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // Compile the code (if there is any) to bitcode. flags := append([]string{"-c", "-emit-llvm", "-o", f.Name() + ".bc", f.Name()}, pkg.CFlags...) + flags = append(flags, "-working-directory="+tmpdir) + if config.TrimPath() { + flags = append(flags, + "-ffile-prefix-map="+tmpdir+"="+config.CSourcePath(pkg.RecordedDir()), + ) + } if config.Options.PrintCommands != nil { config.Options.PrintCommands("clang", flags...) } @@ -633,7 +648,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe defer irbuilder.Dispose() irbuilder.SetInsertPointAtEnd(block) if config.Debug() && !config.Options.SkipDWARF { - addInitAllDebugInfo(mod, llvmInitFn, irbuilder, program) + pos := program.Fset.Position(program.ImportedPackage("runtime").Members["initAll"].Pos()) + pos.Filename = lprogram.Packages["runtime"].RecordedPath(pos.Filename) + addInitAllDebugInfo(mod, llvmInitFn, irbuilder, pos, config.TrimPath()) } ptrType := llvm.PointerType(mod.Context().Int8Type(), 0) for _, pkg := range lprogram.Sorted() { @@ -802,7 +819,13 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe job := &compileJob{ description: "compile extra file " + path, run: func(job *compileJob) error { - result, err := compileAndCacheCFile(abspath, tmpdir, config.CFlags(false), config.Options.PrintCommands) + var compileConfig *cFileCompileConfig + if config.TrimPath() { + compileConfig = &cFileCompileConfig{ + recordedPath: config.CSourcePath(filepath.Join("github.com/tinygo-org/tinygo", path)), + } + } + result, err := compileAndCacheCFile(abspath, tmpdir, config.CFlags(false), compileConfig, config.Options.PrintCommands) job.result = result return err }, @@ -819,7 +842,14 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe job := &compileJob{ description: "compile CGo file " + abspath, run: func(job *compileJob) error { - result, err := compileAndCacheCFile(abspath, tmpdir, pkg.CFlags, config.Options.PrintCommands) + cflags := pkg.CFlags + var compileConfig *cFileCompileConfig + if config.TrimPath() { + compileConfig = &cFileCompileConfig{ + recordedPath: config.CSourcePath(pkg.RecordedPath(abspath)), + } + } + result, err := compileAndCacheCFile(abspath, tmpdir, cflags, compileConfig, config.Options.PrintCommands) job.result = result return err }, @@ -872,11 +902,18 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe description: "link", dependencies: linkerDependencies, run: func(job *compileJob) error { - for _, dependency := range job.dependencies { + for i, dependency := range job.dependencies { if dependency.result == "" { return errors.New("dependency without result: " + dependency.description) } - ldflags = append(ldflags, dependency.result) + linkerInput := dependency.result + if config.TrimPath() && config.LinkerFlavor() == "darwin" { + linkerInput = filepath.Join(tmpdir, fmt.Sprintf("link-input-%d%s", i, filepath.Ext(linkerInput))) + if err := linkOrCopyFile(dependency.result, linkerInput); err != nil { + return err + } + } + ldflags = append(ldflags, linkerInput) } ldflags = append(ldflags, "-mllvm", "-mcpu="+config.CPU()) ldflags = append(ldflags, "-mllvm", "-mattr="+config.Features()) // needed for MIPS softfloat @@ -891,6 +928,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe ldflags = append(ldflags, "--lto-O"+strconv.Itoa(speedLevel), "-cache_path_lto", filepath.Join(cacheDir, "thinlto")) + if config.TrimPath() { + ldflags = append(ldflags, "-oso_prefix", tmpdir+string(filepath.Separator)) + } case "gnu": // Options for the ELF linker. ldflags = append(ldflags, @@ -1139,6 +1179,34 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe return result, nil } +func linkOrCopyFile(src, dst string) error { + if err := os.Link(src, dst); err == nil { + return nil + } + + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return err + } + _, copyErr := io.Copy(destination, source) + closeErr := destination.Close() + if copyErr != nil { + os.Remove(dst) + return copyErr + } + if closeErr != nil { + os.Remove(dst) + return closeErr + } + return nil +} + // createEmbedObjectFile creates a new object file with the given contents, for // the embed package. func createEmbedObjectFile(data, hexSum, sourceFile, sourceDir, tmpdir string, compilerConfig *compiler.Config) (string, error) { @@ -1228,7 +1296,8 @@ func createEmbedObjectFile(data, hexSum, sourceFile, sourceDir, tmpdir string, c return "", err } defer machine.Dispose() - outfile, err := os.CreateTemp(tmpdir, "embed-"+hexSum+"-*.o") + sourcePathHash := sha256.Sum256([]byte(filepath.ToSlash(filepath.Join(sourceDir, sourceFile)))) + outfile, err := os.Create(filepath.Join(tmpdir, "embed-"+hexSum+"-"+hex.EncodeToString(sourcePathHash[:8])+".o")) if err != nil { return "", err } @@ -1281,15 +1350,18 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error { // addInitAllDebugInfo gives runtime.initAll a subprogram so that code emitted // into it by interp or the inliner keeps its line information. -func addInitAllDebugInfo(mod llvm.Module, fn llvm.Value, irbuilder llvm.Builder, program *ssa.Program) { - pos := program.Fset.Position(program.ImportedPackage("runtime").Members["initAll"].Pos()) +func addInitAllDebugInfo(mod llvm.Module, fn llvm.Value, irbuilder llvm.Builder, pos token.Position, trimPath bool) { dir, file := filepath.Split(pos.Filename) + compileDir, compileFile := filepath.Clean(dir), file + if trimPath { + compileDir, compileFile = "", pos.Filename + } dibuilder := llvm.NewDIBuilder(mod) defer dibuilder.Destroy() dibuilder.CreateCompileUnit(llvm.DICompileUnit{ Language: 0xb, // DW_LANG_C99 (0xc, off-by-one?) - File: file, - Dir: filepath.Clean(dir), + File: compileFile, + Dir: compileDir, Producer: "TinyGo", Optimized: true, }) diff --git a/builder/cc.go b/builder/cc.go index 9cc03790b6..6bf3a99225 100644 --- a/builder/cc.go +++ b/builder/cc.go @@ -12,6 +12,7 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "sort" "strings" @@ -21,6 +22,10 @@ import ( "tinygo.org/x/go-llvm" ) +type cFileCompileConfig struct { + recordedPath string +} + // compileAndCacheCFile compiles a C or assembly file using a build cache. // Compiling the same file again (if nothing changed, including included header // files) the output is loaded from the build cache instead. @@ -56,7 +61,7 @@ import ( // depfile but without invalidating its name. For this reason, the depfile is // written on each new compilation (even when it seems unnecessary). However, it // could in rare cases lead to a stale file fetched from the cache. -func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands func(string, ...string)) (string, error) { +func compileAndCacheCFile(abspath, tmpdir string, cflags []string, compileConfig *cFileCompileConfig, printCommands func(string, ...string)) (string, error) { // Hash input file. fileHash, err := hashFile(abspath) if err != nil { @@ -68,16 +73,22 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands defer unlock() // Create cache key for the dependencies file. + recordedPath := "" + if compileConfig != nil { + recordedPath = compileConfig.recordedPath + } buf, err := json.Marshal(struct { - Path string - Hash string - Flags []string - LLVMVersion string + Path string + Hash string + Flags []string + RecordedPath string + LLVMVersion string }{ - Path: abspath, - Hash: fileHash, - Flags: cflags, - LLVMVersion: llvm.Version, + Path: abspath, + Hash: fileHash, + Flags: cflags, + RecordedPath: recordedPath, + LLVMVersion: llvm.Version, }) if err != nil { panic(err) // shouldn't happen @@ -124,7 +135,17 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands depTmpFile.Close() flags := append([]string{}, cflags...) // copy cflags flags = append(flags, "-MD", "-MV", "-MTdeps", "-MF", depTmpFile.Name(), "-flto=thin") // autogenerate dependencies - flags = append(flags, "-c", "-o", objTmpFile.Name(), abspath) + // Match cmd/go/internal/work/exec.go's ccompile directory in both modes. + workingDir := filepath.Dir(abspath) + flags = append(flags, "-working-directory="+workingDir) + if compileConfig != nil { + seed := sha512.Sum512_224([]byte(compileConfig.recordedPath)) + flags = append(flags, + "-fdebug-compilation-dir="+path.Dir(compileConfig.recordedPath), + "-frandom-seed="+hex.EncodeToString(seed[:]), + ) + } + flags = append(flags, "-c", "-o", objTmpFile.Name(), filepath.Base(abspath)) if strings.ToLower(filepath.Ext(abspath)) == ".s" { // If this is an assembly file (.s or .S, lowercase or uppercase), then // we'll need to add -Qunused-arguments because many parameters are @@ -145,6 +166,11 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands if err != nil { return "", err } + for i, dependencyPath := range dependencyPaths { + if !filepath.IsAbs(dependencyPath) { + dependencyPaths[i] = filepath.Join(workingDir, dependencyPath) + } + } dependencyPaths = append(dependencyPaths, abspath) // necessary for .s files dependencySet := make(map[string]struct{}, len(dependencyPaths)) var dependencySlice []string diff --git a/builder/library.go b/builder/library.go index 722f3c8371..8a76950499 100644 --- a/builder/library.go +++ b/builder/library.go @@ -123,6 +123,9 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ } remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name) + if config.TrimPath() { + remapDir = config.CSourcePath(filepath.Join("github.com/tinygo-org/tinygo", ".tinygo-build", l.name)) + } dir := filepath.Join(tmpdir, "build-lib-"+l.name) err = os.Mkdir(dir, 0777) if err != nil { @@ -134,6 +137,13 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ // reproducible. Otherwise the temporary directory is stored in the archive // itself, which varies each run. args := append(l.cflags(target, headerPath), "-c", "-Oz", "-gdwarf-4", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+compileopts.ClangTriple(target), "-fdebug-prefix-map="+dir+"="+remapDir) + if config.TrimPath() { + args = append(args, + "-ffile-prefix-map="+goenv.Get("TINYGOROOT")+"="+config.CSourcePath("github.com/tinygo-org/tinygo"), + "-ffile-prefix-map="+goenv.Get("GOCACHE")+"="+config.CSourcePath("tinygo-cache"), + "-fdebug-compilation-dir=.", + ) + } resourceDir := goenv.ClangResourceDir(false) if resourceDir != "" { args = append(args, "-resource-dir="+resourceDir) diff --git a/builder/sizes.go b/builder/sizes.go index 98a830d9d2..1f8e639c31 100644 --- a/builder/sizes.go +++ b/builder/sizes.go @@ -17,6 +17,7 @@ import ( "strings" "github.com/aykevl/go-wasm" + "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/goenv" ) @@ -946,6 +947,17 @@ func readSection(section memorySection, addresses []addressLine, program *progra // findPackagePath returns the Go package (or a pseudo package) for the given // path. It uses some heuristics, for example for some C libraries. func findPackagePath(path string, packagePathMap map[string]string) (packagePath, filename string) { + for _, prefix := range []string{compileopts.CSourcePathRoot + "/", compileopts.CSourcePathRootWindows + "/"} { + if strings.HasPrefix(filepath.ToSlash(path), prefix) { + recordedPath := strings.TrimPrefix(filepath.ToSlash(path), prefix) + if rel, ok := strings.CutPrefix(recordedPath, "github.com/tinygo-org/tinygo/"); ok { + path = filepath.Join(goenv.Get("TINYGOROOT"), filepath.FromSlash(rel)) + } else { + path = filepath.FromSlash(recordedPath) + } + break + } + } // Check whether this path is part of one of the compiled packages. packagePath, ok := packagePathMap[filepath.Dir(path)] if ok { diff --git a/builder/sizes_test.go b/builder/sizes_test.go index 270a8a3afc..c1e6cabce7 100644 --- a/builder/sizes_test.go +++ b/builder/sizes_test.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "regexp" "runtime" "strconv" @@ -12,6 +13,7 @@ import ( "time" "github.com/tinygo-org/tinygo/compileopts" + "github.com/tinygo-org/tinygo/goenv" ) var sema = make(chan struct{}, runtime.NumCPU()) @@ -24,6 +26,29 @@ type sizeTest struct { opt string } +func TestTrimPathPackageSize(t *testing.T) { + root := goenv.Get("TINYGOROOT") + packages := map[string]string{ + filepath.FromSlash("example.com/main"): "main", + filepath.FromSlash("example.com/dependency@v1.2.3/subpackage"): "example.com/dependency/subpackage", + filepath.Join(root, "src", "runtime"): "runtime", + } + for _, tc := range []struct{ path, pkg, file string }{ + {"example.com/main/main.go", "main", "main.go"}, + {"/_/example.com/main/main.c", "main", "main.c"}, + {"//_/_/example.com/main/main.c", "main", "main.c"}, + {"/_/example.com/dependency@v1.2.3/subpackage/dependency.c", "example.com/dependency/subpackage", "dependency.c"}, + {"/_/github.com/tinygo-org/tinygo/src/runtime/runtime_unix.c", "runtime", "runtime_unix.c"}, + {"/_/github.com/tinygo-org/tinygo/lib/musl/src/stdio/puts.c", "C musl", filepath.FromSlash("src/stdio/puts.c")}, + {"//_/_/github.com/tinygo-org/tinygo/lib/bdwgc/alloc.c", "C bdwgc", "alloc.c"}, + } { + pkg, file := findPackagePath(filepath.FromSlash(tc.path), packages) + if pkg != tc.pkg || file != tc.file { + t.Errorf("findPackagePath(%q) = %q, %q; want %q, %q", tc.path, pkg, file, tc.pkg, tc.file) + } + } +} + // Test whether code and data size is as expected for the given targets. // This tests both the logic of loadProgramSize and checks that code size // doesn't change unintentionally. @@ -116,7 +141,7 @@ func measureBinarySizes(t *testing.T, tests []sizeTest) []*programSize { } func measureBinarySize(tc sizeTest, tmpdir string) (*programSize, error) { - result, err := buildBinaryInDir(tc.target, tc.path, tc.opt, tmpdir) + result, err := buildBinaryInDir(tc.target, tc.path, tc.opt, tmpdir, false) if err != nil { return nil, err } @@ -162,20 +187,25 @@ func formatSizeTable(tests []sizeTest, sizes []*programSize) string { // Check that the -size=full flag attributes binary size to the correct package // without filesystem paths and things like that. func TestSizeFull(t *testing.T) { - tests := []string{ - "microbit", - "wasip1", + tests := []struct { + target string + trimPath bool + }{ + {"microbit", false}, + {"microbit", true}, + {"wasip1", false}, + {"wasip1", true}, } libMatch := regexp.MustCompile(`^C [a-z -]+$`) // example: "C interrupt vector" pkgMatch := regexp.MustCompile(`^[a-z/]+$`) // example: "internal/task" - for _, target := range tests { - t.Run(target, func(t *testing.T) { + for _, test := range tests { + t.Run(fmt.Sprintf("%s/trimpath=%v", test.target, test.trimPath), func(t *testing.T) { t.Parallel() // Build the binary. - result := buildBinary(t, target, "examples/serial") + result := buildBinary(t, test.target, "examples/serial", test.trimPath) // Check whether the binary doesn't contain any unexpected package // names. @@ -200,16 +230,16 @@ func TestSizeFull(t *testing.T) { } } -func buildBinary(t *testing.T, targetString, pkgName string) BuildResult { +func buildBinary(t *testing.T, targetString, pkgName string, trimPath bool) BuildResult { t.Helper() - result, err := buildBinaryInDir(targetString, pkgName, "z", t.TempDir()) + result, err := buildBinaryInDir(targetString, pkgName, "z", t.TempDir(), trimPath) if err != nil { t.Fatal(err) } return result } -func buildBinaryInDir(targetString, pkgName, opt, tmpdir string) (BuildResult, error) { +func buildBinaryInDir(targetString, pkgName, opt, tmpdir string, trimPath bool) (BuildResult, error) { options := compileopts.Options{ Target: targetString, Opt: opt, @@ -217,6 +247,7 @@ func buildBinaryInDir(targetString, pkgName, opt, tmpdir string) (BuildResult, e InterpTimeout: 60 * time.Second, Debug: true, VerifyIR: true, + TrimPath: trimPath, } target, err := compileopts.LoadTarget(&options) if err != nil { diff --git a/compileopts/cflags.go b/compileopts/cflags.go new file mode 100644 index 0000000000..3c620bc842 --- /dev/null +++ b/compileopts/cflags.go @@ -0,0 +1,44 @@ +package compileopts + +import "strings" + +// RewriteCFlags rewrites path operands without changing other compiler options. +func RewriteCFlags(flags []string, rewrite func(string) string) []string { + flags = append([]string(nil), flags...) + nextIsPath := false + for i, flag := range flags { + if nextIsPath { + flags[i] = rewrite(flag) + nextIsPath = false + continue + } + switch flag { + case "-I", "-L", "-F", "-isystem", "-isystem-after", "-iquote", "-idirafter", "-include", "--include", "-imacros", "--imacros", "-include-pch", "-isysroot", "--sysroot", "-resource-dir", "-iframework", "-iframeworkwithsysroot": + nextIsPath = true + continue + } + for _, prefix := range []string{ + "-I", "-L", "-F", + "-isystem-after", "-isystem", "-iquote", "-idirafter", + "-iframeworkwithsysroot", "-iframework", + "-include", "--include", "-imacros", "--imacros", + "--sysroot=", "-isysroot=", "-isysroot", "-resource-dir=", + "-fmodule-map-file=", "-fmodules-cache-path=", + } { + if strings.HasPrefix(flag, prefix) { + flags[i] = prefix + rewrite(strings.TrimPrefix(flag, prefix)) + break + } + } + for _, prefix := range []string{"-fdebug-prefix-map=", "-ffile-prefix-map=", "-fmacro-prefix-map="} { + if strings.HasPrefix(flag, prefix) { + mapping := strings.TrimPrefix(flag, prefix) + if sep := strings.LastIndexByte(mapping, '='); sep >= 0 { + flags[i] = prefix + rewrite(mapping[:sep]) + mapping[sep:] + } + break + } + } + } + return flags +} diff --git a/compileopts/config.go b/compileopts/config.go index 1426ac424c..b7cf0d5ae7 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "path" "path/filepath" "regexp" "slices" @@ -59,6 +60,26 @@ func (c *Config) BuildMode() string { return "default" } +// TrimPath reports whether local file system paths must be removed from the +// output. +func (c *Config) TrimPath() bool { + return c.Options.TrimPath +} + +const ( + CSourcePathRoot = "/_" + CSourcePathRootWindows = "//_/_" +) + +// CSourcePath avoids relative DWARF paths being joined to DW_AT_comp_dir. +// See go.dev/src/cmd/go/internal/work/exec.go (Builder.ccompile). +func (c *Config) CSourcePath(filename string) string { + if c.GOOS() == "windows" { + return "/" + path.Join(CSourcePathRootWindows, filepath.ToSlash(filename)) + } + return path.Join(CSourcePathRoot, filepath.ToSlash(filename)) +} + // Features returns a list of features this CPU supports. For example, for a // RISC-V processor, that could be "+a,+c,+m". For many targets, an empty list // will be returned. @@ -336,6 +357,9 @@ func (c *Config) LibraryPath(name string) string { if c.LibcNeedsMalloc() { options += "+malloc" } + if c.TrimPath() { + options += "+trimpath-v1" + } // No precompiled library found. Determine the path name that will be used // in the build cache. @@ -388,6 +412,13 @@ func (c *Config) CFlags(libclang bool) []string { "-I"+filepath.Join(goenv.Get("TINYGOROOT"), "lib", "bdwgc", "include"), ) } + if c.TrimPath() { + cflags = append(cflags, + "-ffile-prefix-map="+goenv.Get("TINYGOROOT")+"="+c.CSourcePath("github.com/tinygo-org/tinygo"), + "-ffile-prefix-map="+goenv.Get("GOCACHE")+"="+c.CSourcePath("tinygo-cache"), + "-fdebug-compilation-dir=.", + ) + } // Always emit debug information. It is optionally stripped at link time. cflags = append(cflags, "-gdwarf-4") // Use the same optimization level as TinyGo. diff --git a/compileopts/options.go b/compileopts/options.go index 750dcb42dc..ad23c38471 100644 --- a/compileopts/options.go +++ b/compileopts/options.go @@ -31,6 +31,7 @@ type Options struct { Directory string // working dir, leave it unset to use the current working dir Target string BuildMode string // -buildmode flag + TrimPath bool // -trimpath flag Opt string GC string PanicStrategy string diff --git a/compiler/compiler.go b/compiler/compiler.go index 84aabd5dee..7914c0ade5 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -50,6 +50,7 @@ type Config struct { RelocationModel string SizeLevel int TinyGoVersion string // for llvm.ident + TrimPath bool // Various compiler options that determine how code is generated. Scheduler string @@ -861,7 +862,8 @@ func (c *compilerContext) attachDebugInfoRawWithDefinition(f *ssa.Function, llvm // one. func (c *compilerContext) getDIFile(filename string) llvm.Metadata { if _, ok := c.difiles[filename]; !ok { - dir, file := filepath.Split(filename) + recordedPath := c.loaderPkg.RecordedPath(filename) + dir, file := filepath.Split(recordedPath) if dir != "" { dir = dir[:len(dir)-1] } diff --git a/loader/loader.go b/loader/loader.go index df46d662fa..b21099d8d5 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -365,6 +365,168 @@ func (p *Package) OriginalDir() string { return strings.TrimSuffix(p.program.getOriginalPath(p.Dir+string(os.PathSeparator)), string(os.PathSeparator)) } +// OriginalModuleDir returns the real module directory. +func (p *Package) OriginalModuleDir() string { + if p.Module.Dir != "" { + return strings.TrimSuffix(p.program.getOriginalPath(p.Module.Dir+string(os.PathSeparator)), string(os.PathSeparator)) + } + if p.Module.Path == "" && p.Root != "" { + root := filepath.Join(p.Root, "src") + return strings.TrimSuffix(p.program.getOriginalPath(root+string(os.PathSeparator)), string(os.PathSeparator)) + } + if p.Module.Path != "" { + importPath := p.sourceImportPath() + rel := "." + if importPath != p.Module.Path { + rel = strings.TrimPrefix(importPath, p.Module.Path+"/") + } + if rel == "." || rel != importPath { + dir := p.OriginalDir() + if rel != "." { + for range strings.Split(rel, "/") { + dir = filepath.Dir(dir) + } + } + return dir + } + } + return p.OriginalDir() +} + +// RecordedModuleDir returns the module directory to record in the output. +func (p *Package) RecordedModuleDir() string { + if !p.program.config.TrimPath() { + return p.OriginalModuleDir() + } + if p.Module.Path == "" { + if p.Root != "" { + return "." + } + return p.RecordedDir() + } + if p.Module.Version != "" { + return p.Module.Path + "@" + p.Module.Version + } + return p.Module.Path +} + +// RecordedDir returns the source directory to record in the output. +func (p *Package) RecordedDir() string { + if !p.program.config.TrimPath() { + return p.OriginalDir() + } + if p.Module.Path != "" && p.Module.Version != "" { + return path.Join(p.Module.Path+"@"+p.Module.Version, strings.TrimPrefix(p.sourceImportPath(), p.Module.Path)) + } + return p.sourceImportPath() +} + +func (p *Package) sourceImportPath() string { + if p.ForTest != "" && p.ImportPath == p.ForTest+"_test" { + return p.ForTest + } + if p.program.config.TestConfig.CompileTestBinary && p == p.program.MainPkg() { + return strings.TrimSuffix(p.ImportPath, ".test") + } + return p.ImportPath +} + +// RecordedPath returns the source path to record in the output. +func (p *Package) RecordedPath(filename string) string { + if !p.program.config.TrimPath() { + return filename + } + if filepath.Base(filename) == "!cgo.go" { + return path.Join(p.RecordedDir(), "!cgo.go") + } + if p.program.config.TestConfig.CompileTestBinary { + for _, generated := range p.program.MainPkg().GoFiles { + if filename == generated { + return "_testmain.go" + } + } + } + for _, root := range []string{ + filepath.Join(p.program.goroot, "src"), + filepath.Join(goenv.Get("GOROOT"), "src"), + filepath.Join(goenv.Get("TINYGOROOT"), "src"), + } { + rel, err := filepath.Rel(root, filename) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return filepath.ToSlash(rel) + } + } + + var recordedPath string + var matchedDirLength int + for _, sourcePkg := range p.program.Packages { + dir := sourcePkg.OriginalDir() + rel, err := filepath.Rel(dir, filename) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + continue + } + candidate := path.Join(sourcePkg.RecordedDir(), filepath.ToSlash(rel)) + if len(dir) > matchedDirLength || len(dir) == matchedDirLength && candidate < recordedPath { + matchedDirLength = len(dir) + recordedPath = candidate + } + } + if recordedPath != "" { + return recordedPath + } + for _, sourcePkg := range p.program.Packages { + dir := sourcePkg.OriginalModuleDir() + rel, err := filepath.Rel(dir, filename) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + continue + } + candidate := path.Join(sourcePkg.RecordedModuleDir(), filepath.ToSlash(rel)) + if len(dir) > matchedDirLength || len(dir) == matchedDirLength && candidate < recordedPath { + matchedDirLength = len(dir) + recordedPath = candidate + } + } + if recordedPath != "" { + return recordedPath + } + if filepath.IsAbs(filename) { + return filename + } + return filepath.ToSlash(filename) +} + +// DebugPrefixMap returns the Clang flag that maps this package to its recorded +// source directory. +func (p *Package) DebugPrefixMap() string { + return "-ffile-prefix-map=" + p.OriginalModuleDir() + "=" + p.program.config.CSourcePath(p.RecordedModuleDir()) +} + +// RecordedCFlags returns C flags with path options rewritten to use recorded +// paths. +func (p *Package) RecordedCFlags() []string { + if !p.program.config.TrimPath() { + return p.CFlags + } + return compileopts.RewriteCFlags(p.CFlags, p.recordedCFlagPath) +} + +func (p *Package) recordedCFlagPath(filename string) string { + if recorded := p.RecordedPath(filename); recorded != filename { + return recorded + } + for _, root := range []struct{ from, to string }{ + {p.OriginalModuleDir(), p.RecordedModuleDir()}, + {goenv.Get("TINYGOROOT"), "github.com/tinygo-org/tinygo"}, + {goenv.Get("GOCACHE"), "tinygo-cache"}, + } { + rel, err := filepath.Rel(root.from, filename) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return path.Join(root.to, filepath.ToSlash(rel)) + } + } + return filename +} + // parseFile is a wrapper around parser.ParseFile. func (p *Package) parseFile(path string, mode parser.Mode) (*ast.File, error) { originalPath := p.program.getOriginalPath(path) @@ -513,7 +675,16 @@ func (p *Package) parseFiles() ([]*ast.File, error) { return nil, fmt.Errorf("failed to split CGO_CFLAGS: %w", err) } initialCFlags = append(initialCFlags, cgoCFlags...) - generated, headerCode, cflags, ldflags, accessedFiles, errs := cgo.Process(files, p.program.workingDir, p.ImportPath, p.program.fset, initialCFlags, p.program.config.GOOS()) + if p.program.config.TrimPath() { + initialCFlags = append(initialCFlags, + p.DebugPrefixMap(), + "-ffile-prefix-map="+filepath.Join(p.program.goroot, "src")+"="+p.program.config.CSourcePath("."), + ) + } + // LLVM 15's driver changes the process cwd. Set only FileManager's cwd. + // See clang/lib/Driver/Driver.cpp and clang/lib/Basic/FileManager.cpp. + parseFlags := append(append([]string(nil), initialCFlags...), "-Xclang", "-working-directory", "-Xclang", p.Dir) + generated, headerCode, cflags, ldflags, accessedFiles, errs := cgo.Process(files, p.program.workingDir, p.ImportPath, p.program.fset, parseFlags, p.program.config.GOOS()) p.CFlags = append(initialCFlags, cflags...) p.CGoHeaders = headerCode for path, hash := range accessedFiles { diff --git a/loader/loader_test.go b/loader/loader_test.go new file mode 100644 index 0000000000..4ec5a04c54 --- /dev/null +++ b/loader/loader_test.go @@ -0,0 +1,131 @@ +package loader + +import ( + "path/filepath" + "slices" + "testing" + + "github.com/tinygo-org/tinygo/compileopts" + "github.com/tinygo-org/tinygo/goenv" +) + +func TestRecordedPath(t *testing.T) { + config := &compileopts.Config{ + Options: &compileopts.Options{TrimPath: true}, + Target: &compileopts.TargetSpec{GOOS: "linux"}, + } + program := &Program{ + config: config, + goroot: filepath.FromSlash("/goroot"), + Packages: make(map[string]*Package), + } + pkg := &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash("/tmp/module"), + ImportPath: "example.com/main", + }, + } + pkg.Module.Path = "example.com/main" + pkg.Module.Dir = filepath.FromSlash("/tmp/module") + dependency := &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash("/tmp/dependency/subpackage"), + ImportPath: "example.com/dependency/subpackage", + }, + } + dependency.Module.Path = "example.com/dependency" + dependency.Module.Version = "v1.2.3" + dependency.Module.Dir = filepath.FromSlash("/tmp/dependency") + dependency.CFlags = []string{ + "-I" + filepath.FromSlash("/tmp/dependency/include"), + "-isystem", + filepath.FromSlash("/tmp/dependency/system"), + "-iquote" + filepath.FromSlash("/tmp/dependency/include"), + "-ffile-prefix-map=" + filepath.FromSlash("/tmp/dependency") + "=/mapped", + `-DCONFIG_PATH="/tmp/dependency"`, + } + program.Packages[pkg.ImportPath] = pkg + program.Packages[dependency.ImportPath] = dependency + + if got, want := dependency.RecordedDir(), "example.com/dependency@v1.2.3/subpackage"; got != want { + t.Fatalf("RecordedDir() = %q, want %q", got, want) + } + filename := filepath.FromSlash("/tmp/dependency/subpackage/file.go") + if got, want := pkg.RecordedPath(filename), "example.com/dependency@v1.2.3/subpackage/file.go"; got != want { + t.Fatalf("RecordedPath() = %q, want %q", got, want) + } + header := filepath.FromSlash("/tmp/dependency/include/shared.h") + if got, want := pkg.RecordedPath(header), "example.com/dependency@v1.2.3/include/shared.h"; got != want { + t.Fatalf("RecordedPath() = %q, want %q", got, want) + } + if got, want := dependency.DebugPrefixMap(), "-ffile-prefix-map="+filepath.FromSlash("/tmp/dependency")+"=/_/example.com/dependency@v1.2.3"; got != want { + t.Fatalf("DebugPrefixMap() = %q, want %q", got, want) + } + if got, want := dependency.RecordedCFlags(), []string{ + "-Iexample.com/dependency@v1.2.3/include", + "-isystem", + "example.com/dependency@v1.2.3/system", + "-iquoteexample.com/dependency@v1.2.3/include", + "-ffile-prefix-map=example.com/dependency@v1.2.3=/mapped", + `-DCONFIG_PATH="/tmp/dependency"`, + }; !slices.Equal(got, want) { + t.Fatalf("RecordedCFlags() = %q, want %q", got, want) + } + + for _, root := range []string{program.goroot, goenv.Get("GOROOT"), goenv.Get("TINYGOROOT")} { + filename := filepath.Join(root, "src", "runtime", "header.h") + if got := pkg.RecordedPath(filename); got != "runtime/header.h" { + t.Errorf("RecordedPath(%q) = %q", filename, got) + } + if got := pkg.recordedCFlagPath(filename); got != "runtime/header.h" { + t.Errorf("recordedCFlagPath(%q) = %q", filename, got) + } + } + outside := filepath.FromSlash("/tmp/module-other/include") + if got := pkg.recordedCFlagPath(outside); got != outside { + t.Errorf("rewrote a path outside the module: %q", got) + } + + vendored := &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash("/tmp/main/vendor/example.com/dependency/subpackage"), + ImportPath: "example.com/dependency/subpackage", + }, + } + vendored.Module.Path = "example.com/dependency" + vendored.Module.Version = "v1.2.3" + program.Packages[vendored.ImportPath] = vendored + if got, want := vendored.OriginalModuleDir(), filepath.FromSlash("/tmp/main/vendor/example.com/dependency"); got != want { + t.Fatalf("vendored OriginalModuleDir() = %q, want %q", got, want) + } + vendoredHeader := filepath.FromSlash("/tmp/main/vendor/example.com/dependency/include/shared.h") + if got, want := pkg.RecordedPath(vendoredHeader), "example.com/dependency@v1.2.3/include/shared.h"; got != want { + t.Fatalf("vendored RecordedPath() = %q, want %q", got, want) + } + if got, want := vendored.DebugPrefixMap(), "-ffile-prefix-map="+filepath.FromSlash("/tmp/main/vendor/example.com/dependency")+"=/_/example.com/dependency@v1.2.3"; got != want { + t.Fatalf("vendored DebugPrefixMap() = %q, want %q", got, want) + } + + gopath := &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash("/gopath/src/example.com/dependency/subpackage"), + ImportPath: "example.com/dependency/subpackage", + Root: filepath.FromSlash("/gopath"), + }, + } + program.Packages[gopath.ImportPath] = gopath + if got, want := gopath.OriginalModuleDir(), filepath.FromSlash("/gopath/src"); got != want { + t.Fatalf("GOPATH OriginalModuleDir() = %q, want %q", got, want) + } + gopathHeader := filepath.FromSlash("/gopath/src/example.com/dependency/include/shared.h") + if got, want := pkg.RecordedPath(gopathHeader), "example.com/dependency/include/shared.h"; got != want { + t.Fatalf("GOPATH RecordedPath() = %q, want %q", got, want) + } + if got, want := gopath.DebugPrefixMap(), "-ffile-prefix-map="+filepath.FromSlash("/gopath/src")+"=/_"; got != want { + t.Fatalf("GOPATH DebugPrefixMap() = %q, want %q", got, want) + } +} diff --git a/main.go b/main.go index 321e0c3979..7e1dfee78e 100644 --- a/main.go +++ b/main.go @@ -1785,6 +1785,7 @@ func main() { target := flag.String("target", "", "chip/board name or JSON target specification file") buildMode := flag.String("buildmode", "", "build mode to use (default, c-shared, wasi-legacy)") buildVCS := flag.String("buildvcs", "auto", "whether to stamp version control information (true, false, auto)") + trimPath := flag.Bool("trimpath", false, "remove all file system paths from the resulting executable") var stackSize uint64 flag.Func("stack-size", "goroutine stack size (if unknown at compile time)", func(s string) error { size, err := bytesize.Parse(s) @@ -1920,6 +1921,7 @@ func main() { Target: *target, BuildMode: *buildMode, BuildVCS: *buildVCS, + TrimPath: *trimPath, StackSize: stackSize, Opt: *opt, GC: *gc, diff --git a/main_test.go b/main_test.go index d752baa0df..2a915347c0 100644 --- a/main_test.go +++ b/main_test.go @@ -9,14 +9,18 @@ import ( "context" "debug/dwarf" "debug/elf" + "debug/pe" "errors" "flag" + "fmt" "io" "os" "os/exec" + "path/filepath" "reflect" "regexp" "runtime" + "slices" "strings" "sync" "testing" @@ -50,6 +54,265 @@ var supportedLinuxArches = map[string]string{ var sema = make(chan struct{}, runtime.NumCPU()) +func TestTrimPath(t *testing.T) { + t.Setenv("CGO_CFLAGS", "-iquoteinclude -includestdint.h -imacros relative.h") + root := t.TempDir() + var binaries [][]byte + options := optionsFromTarget(*testTarget, sema) + config, err := builder.NewConfig(&options) + if err != nil { + t.Fatal(err) + } + + for _, name := range []string{"a", "b"} { + dir := filepath.Join(root, name) + if err := os.CopyFS(dir, os.DirFS(filepath.Join(TESTDATA, "trimpath"))); err != nil { + t.Fatal(err) + } + + outpath := filepath.Join(root, name+".out") + cache := filepath.Join(root, "cache-"+name) + cmd := trimPathCommand(t, dir, "build", "-trimpath", "-p=4", "-o", outpath, ".") + cmd.Env = append(cmd.Env, "XDG_CACHE_HOME="+cache, "HOME="+cache, "LocalAppData="+cache) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build failed: %v\n%s", err, output) + } + binary, err := os.ReadFile(outpath) + if err != nil { + t.Fatal(err) + } + for _, localPath := range []string{root, goenv.Get("GOROOT"), goenv.Get("TINYGOROOT"), goenv.Get("GOCACHE")} { + if bytes.Contains(binary, []byte(localPath)) || bytes.Contains(binary, []byte(filepath.ToSlash(localPath))) { + t.Errorf("trimmed binary contains local path %q", localPath) + } + } + binaries = append(binaries, binary) + } + + if !bytes.Equal(binaries[0], binaries[1]) { + t.Error("trimmed binaries built in different directories are not identical") + } + if config.GOOS() != "darwin" { + files := trimPathDWARFFiles(t, filepath.Join(root, "a.out"), config.GOOS()) + headerPath := "/_/example.com/dependency@v1.2.3/include/shared.h" + if config.GOOS() == "windows" { + headerPath = "//_/_/example.com/dependency@v1.2.3/include/shared.h" + } + for _, want := range []string{ + "example.com/trimpath/main.go", + "example.com/dependency@v1.2.3/subpackage/dependency.go", + headerPath, + } { + if !slices.Contains(files, want) { + t.Errorf("missing DWARF path %q in %v", want, files) + } + } + for _, file := range files { + if strings.Contains(file, filepath.ToSlash(root)) || strings.Count(file, "example.com/") > 1 { + t.Errorf("invalid DWARF path %q", file) + } + } + } + + dir := filepath.Join(root, "a") + for _, trim := range []bool{false, true} { + outpath := filepath.Join(root, "nodebug.out") + args := []string{"build", "-no-debug", "-o", outpath} + if trim { + args = append(args, "-trimpath") + } + cmd := trimPathCommand(t, dir, append(args, ".")...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build failed: %v\n%s", err, output) + } + binary, err := os.ReadFile(outpath) + if err != nil { + t.Fatal(err) + } + containsPath := bytes.Contains(binary, []byte(dir)) || bytes.Contains(binary, []byte(filepath.ToSlash(dir))) + if containsPath == trim { + t.Errorf("trimpath=%v: contains local __FILE__ path=%v", trim, containsPath) + } + } +} + +func TestTrimPathStackSizes(t *testing.T) { + dir := t.TempDir() + source := `package main +func main() { + done := make(chan int) + go func() { done <- 42 }() + println(<-done) +} +` + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte(source), 0o600); err != nil { + t.Fatal(err) + } + for _, trim := range []bool{false, true} { + t.Run(fmt.Sprintf("trimpath=%v", trim), func(t *testing.T) { + outpath := filepath.Join(dir, "main.elf") + args := []string{"build", "-target=pico2", "-o", outpath} + if trim { + args = append(args, "-trimpath") + } + cmd := trimPathCommand(t, dir, append(args, "main.go")...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build failed: %v\n%s", err, output) + } + file, err := elf.Open(outpath) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if section := file.Section(".tinygo_stacksizes"); section == nil || section.Size == 0 { + t.Fatal("missing automatic stack-size data") + } + binary, err := os.ReadFile(outpath) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(binary, []byte("task_stack.go")) { + t.Fatal("missing stack-size source file") + } + sourceDir := filepath.Join(goenv.Get("TINYGOROOT"), "src") + containsPath := bytes.Contains(binary, []byte(sourceDir)) || bytes.Contains(binary, []byte(filepath.ToSlash(sourceDir))) + if containsPath == trim { + t.Errorf("trimpath=%v: contains local runtime source directory=%v", trim, containsPath) + } + }) + } +} + +func trimPathCommand(t *testing.T, dir string, args ...string) *exec.Cmd { + t.Helper() + options := optionsFromTarget(*testTarget, sema) + if options.Target != "" { + args = slices.Insert(args, 1, "-target="+options.Target) + } + cmd := exec.Command(os.Args[0], append([]string{"test-main"}, args...)...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "TINYGOROOT="+goenv.Get("TINYGOROOT"), + "GOPATH="+goenv.Get("GOPATH"), + "GOOS="+options.GOOS, + "GOARCH="+options.GOARCH, + "GOARM="+options.GOARM, + ) + return cmd +} + +func trimPathDWARFFiles(t *testing.T, filename, goos string) []string { + t.Helper() + f, err := os.Open(filename) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var data *dwarf.Data + switch goos { + case "windows": + file, parseErr := pe.NewFile(f) + if parseErr != nil { + t.Fatal(parseErr) + } + data, err = file.DWARF() + case "wasip1", "wasip2", "js": + file, parseErr := wasm.Parse(f) + if parseErr != nil { + t.Fatal(parseErr) + } + data, err = file.DWARF() + default: + file, parseErr := elf.NewFile(f) + if parseErr != nil { + t.Fatal(parseErr) + } + data, err = file.DWARF() + } + if err != nil { + t.Fatal(err) + } + var files []string + reader := data.Reader() + for { + entry, err := reader.Next() + if err != nil { + t.Fatal(err) + } + if entry == nil { + break + } + if entry.Tag != dwarf.TagCompileUnit { + continue + } + lines, err := data.LineReader(entry) + if err != nil { + t.Fatal(err) + } + if lines == nil { + continue + } + var line dwarf.LineEntry + for { + if err := lines.Next(&line); err != nil { + if err != io.EOF { + t.Fatal(err) + } + break + } + } + for _, file := range lines.Files() { + if file != nil { + files = append(files, strings.ReplaceAll(file.Name, "\\", "/")) + } + } + } + return files +} + +func TestTrimPathTestPackages(t *testing.T) { + options := optionsFromTarget(*testTarget, sema) + config, err := builder.NewConfig(&options) + if err != nil { + t.Fatal(err) + } + if config.GOOS() == "darwin" { + t.Skip("Mach-O executables keep DWARF in separate object files") + } + for _, externalOnly := range []bool{false, true} { + dir := t.TempDir() + files := map[string]string{ + "go.mod": "module example.com/pkg_test\n\ngo 1.23\n", + "external_test.go": "package pkg_test\nimport \"testing\"\nfunc TestPath(t *testing.T) { t.Log(\"path\") }\n", + } + if !externalOnly { + files["pkg.go"] = "package pkg\nvar Value = 1\n" + } + for name, data := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil { + t.Fatal(err) + } + } + outpath := filepath.Join(dir, "test.out") + cmd := trimPathCommand(t, dir, "test", "-trimpath", "-c", "-o", outpath, ".") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("test build failed: %v\n%s", err, output) + } + got := trimPathDWARFFiles(t, outpath, config.GOOS()) + if !slices.Contains(got, "example.com/pkg_test/external_test.go") { + t.Errorf("externalOnly=%v: incorrect test paths: %v", externalOnly, got) + } + if !slices.Contains(got, "_testmain.go") { + t.Errorf("externalOnly=%v: missing generated test main path", externalOnly) + } + for _, file := range got { + if filepath.IsAbs(filepath.FromSlash(file)) && !strings.HasPrefix(file, "/_/") && !strings.HasPrefix(file, "//_/_/") { + t.Errorf("externalOnly=%v: untrimmed test path %q", externalOnly, file) + } + } + } +} + func TestBuild(t *testing.T) { t.Parallel() @@ -1573,6 +1836,11 @@ func TestMain(m *testing.M) { os.Exit(1) } os.Exit(0) + case "test-main": + os.Args = append([]string{os.Args[0]}, os.Args[2:]...) + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) + main() + os.Exit(0) } } diff --git a/make/config.mk b/make/config.mk index 5b592d8261..020024e4ae 100644 --- a/make/config.mk +++ b/make/config.mk @@ -36,7 +36,7 @@ GO ?= go # Flags to pass to go test. GOTESTFLAGS ?= -GOTESTPKGS ?= ./builder ./cgo ./compileopts ./compiler ./interp ./transform . +GOTESTPKGS ?= ./builder ./cgo ./compileopts ./compiler ./interp ./loader ./transform . # tinygo binary for tests TINYGO ?= $(call detect,tinygo,tinygo $(CURDIR)/build/tinygo) diff --git a/testdata/trimpath/data.txt b/testdata/trimpath/data.txt new file mode 100644 index 0000000000..80e21b7cb6 --- /dev/null +++ b/testdata/trimpath/data.txt @@ -0,0 +1 @@ +embedded diff --git a/testdata/trimpath/dependency/go.mod b/testdata/trimpath/dependency/go.mod new file mode 100644 index 0000000000..5150d6776e --- /dev/null +++ b/testdata/trimpath/dependency/go.mod @@ -0,0 +1,3 @@ +module example.com/dependency + +go 1.23 diff --git a/testdata/trimpath/dependency/include/shared.h b/testdata/trimpath/dependency/include/shared.h new file mode 100644 index 0000000000..c35cf87dd7 --- /dev/null +++ b/testdata/trimpath/dependency/include/shared.h @@ -0,0 +1,7 @@ +static inline int sharedValueInline(void) { + return 7; +} + +static inline const char *sharedHeaderPathInline(void) { + return __FILE__; +} diff --git a/testdata/trimpath/dependency/subpackage/dependency.c b/testdata/trimpath/dependency/subpackage/dependency.c new file mode 100644 index 0000000000..f6fc682324 --- /dev/null +++ b/testdata/trimpath/dependency/subpackage/dependency.c @@ -0,0 +1,12 @@ +#include "shared.h" +#include "relative.h" + +_Static_assert(RELATIVE_VALUE == 43, "incorrect relative include"); + +int sharedValue(void) { + return sharedValueInline(); +} + +const char *sharedHeaderPath(void) { + return sharedHeaderPathInline(); +} diff --git a/testdata/trimpath/dependency/subpackage/dependency.go b/testdata/trimpath/dependency/subpackage/dependency.go new file mode 100644 index 0000000000..3dac0ee8cc --- /dev/null +++ b/testdata/trimpath/dependency/subpackage/dependency.go @@ -0,0 +1,16 @@ +package dependency + +/* +#cgo CFLAGS: -I../include -Iinclude +#include "relative.h" +_Static_assert(RELATIVE_VALUE == 43, "incorrect relative include"); +int sharedValue(void); +const char *sharedHeaderPath(void); +*/ +import "C" + +//go:noinline +func Value() int { return int(C.sharedValue()) } + +//go:noinline +func HeaderPath() string { return C.GoString(C.sharedHeaderPath()) } diff --git a/testdata/trimpath/dependency/subpackage/include/relative.h b/testdata/trimpath/dependency/subpackage/include/relative.h new file mode 100644 index 0000000000..63cc107be8 --- /dev/null +++ b/testdata/trimpath/dependency/subpackage/include/relative.h @@ -0,0 +1 @@ +#define RELATIVE_VALUE 43 diff --git a/testdata/trimpath/go.mod b/testdata/trimpath/go.mod new file mode 100644 index 0000000000..1c1d89052a --- /dev/null +++ b/testdata/trimpath/go.mod @@ -0,0 +1,7 @@ +module example.com/trimpath + +go 1.23 + +require example.com/dependency v1.2.3 + +replace example.com/dependency => ./dependency diff --git a/testdata/trimpath/include/relative.h b/testdata/trimpath/include/relative.h new file mode 100644 index 0000000000..6c49439eea --- /dev/null +++ b/testdata/trimpath/include/relative.h @@ -0,0 +1 @@ +#define RELATIVE_VALUE 42 diff --git a/testdata/trimpath/main.c b/testdata/trimpath/main.c new file mode 100644 index 0000000000..28b992f873 --- /dev/null +++ b/testdata/trimpath/main.c @@ -0,0 +1,5 @@ +#include "relative.h" +#include + +_Static_assert(RELATIVE_VALUE == 42, "incorrect relative include"); +int32_t value(void) { return RELATIVE_VALUE; } diff --git a/testdata/trimpath/main.go b/testdata/trimpath/main.go new file mode 100644 index 0000000000..4b55529869 --- /dev/null +++ b/testdata/trimpath/main.go @@ -0,0 +1,22 @@ +package main + +/* +#cgo CFLAGS: -Iinclude +int value(void); +static int inlineValue(void) { return 1; } +*/ +import "C" + +import ( + _ "embed" + "runtime" + + "example.com/dependency/subpackage" +) + +//go:embed data.txt +var data string + +func main() { + println(C.value(), C.inlineValue(), dependency.Value(), dependency.HeaderPath(), data, runtime.GOROOT()) +} diff --git a/transform/stacksize.go b/transform/stacksize.go index 2f7a6c1d6f..8d1d72bc6a 100644 --- a/transform/stacksize.go +++ b/transform/stacksize.go @@ -81,9 +81,13 @@ func CreateStackSizeLoads(mod llvm.Module, config *compileopts.Config) []string }, }, }) + sourceDir := "" + if !config.TrimPath() { + sourceDir = filepath.Join(goenv.Get("TINYGOROOT"), "src") + } diglobal := dibuilder.CreateGlobalVariableExpression(llvm.Metadata{}, llvm.DIGlobalVariableExpression{ Name: "internal/task.stackSizes", - File: dibuilder.CreateFile("internal/task/task_stack.go", filepath.Join(goenv.Get("TINYGOROOT"), "src")), + File: dibuilder.CreateFile("internal/task/task_stack.go", sourceDir), Line: 1, Type: ditype, Expr: dibuilder.CreateExpression(nil), From 1d3c8d97c8ba9570076e9b411ad5daa3bf6a0d4f Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:24:04 -0700 Subject: [PATCH 3/6] builder: make trimmed Darwin builds reproducible with LLVM 15 LLD 15 records ThinLTO cache paths and object timestamps in Mach-O outputs. These make trimmed binaries depend on cache location and build time even when their source paths have been removed. LLD 16 changed cached-object handling in https://reviews.llvm.org/D131624. Disable ThinLTO caching only for trimmed Darwin builds on LLVM 15, and suppress object timestamps. Other compilation caches remain enabled. Exercise Darwin trimpath builds in the oldest-toolchain compatibility job. --- .github/workflows/compat.yml | 1 + builder/build.go | 16 ++++++++++++---- builder/darwin-libsystem.go | 2 +- builder/mingw-w64.go | 2 +- builder/tools.go | 3 ++- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml index cc53fae05f..ea7ef41e3a 100644 --- a/.github/workflows/compat.yml +++ b/.github/workflows/compat.yml @@ -69,4 +69,5 @@ jobs: - run: tinygo version - run: make gen-device -j4 - run: go test -tags=llvm${{ env.LLVM }} -short -skip=TestErrors + - run: go test -tags=llvm${{ env.LLVM }} -run TestTrimPath -target=darwin/amd64 - run: make smoketest-quick XTENSA=0 diff --git a/builder/build.go b/builder/build.go index 35cc210dd7..d41062c530 100644 --- a/builder/build.go +++ b/builder/build.go @@ -32,6 +32,7 @@ import ( "github.com/gofrs/flock" "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/compiler" + "github.com/tinygo-org/tinygo/compiler/llvmutil" "github.com/tinygo-org/tinygo/goenv" "github.com/tinygo-org/tinygo/interp" "github.com/tinygo-org/tinygo/loader" @@ -925,9 +926,12 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe "--thinlto-cache-dir="+filepath.Join(cacheDir, "thinlto")) case "darwin": // Options for the ld64-compatible lld linker. - ldflags = append(ldflags, - "--lto-O"+strconv.Itoa(speedLevel), - "-cache_path_lto", filepath.Join(cacheDir, "thinlto")) + ldflags = append(ldflags, "--lto-O"+strconv.Itoa(speedLevel)) + // LLD 15 embeds cache paths in OSO symbols, unlike LLD 16+. + // Fixed in LLD 16 by https://reviews.llvm.org/D131624. + if !config.TrimPath() || llvmutil.Version() >= 16 { + ldflags = append(ldflags, "-cache_path_lto", filepath.Join(cacheDir, "thinlto")) + } if config.TrimPath() { ldflags = append(ldflags, "-oso_prefix", tmpdir+string(filepath.Separator)) } @@ -954,7 +958,11 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe if config.Options.PrintCommands != nil { config.Options.PrintCommands(config.Target.Linker, ldflags...) } - err = link(config.Target.Linker, ldflags...) + var linkEnv []string + if config.TrimPath() && config.LinkerFlavor() == "darwin" { + linkEnv = append(linkEnv, "ZERO_AR_DATE=1") + } + err = link(config.Target.Linker, ldflags, linkEnv...) if err != nil { return err } diff --git a/builder/darwin-libsystem.go b/builder/darwin-libsystem.go index e47b54230e..de927e2eb8 100644 --- a/builder/darwin-libsystem.go +++ b/builder/darwin-libsystem.go @@ -52,7 +52,7 @@ func makeDarwinLibSystemJob(config *compileopts.Config, tmpdir string) *compileJ if config.Options.PrintCommands != nil { config.Options.PrintCommands("ld.lld", flags...) } - return link("ld.lld", flags...) + return link("ld.lld", flags) }, } } diff --git a/builder/mingw-w64.go b/builder/mingw-w64.go index 64c3be3f4c..d0beba9e80 100644 --- a/builder/mingw-w64.go +++ b/builder/mingw-w64.go @@ -165,7 +165,7 @@ func makeMinGWExtraLibs(tmpdir, goarch string) []*compileJob { return err } } - return link("ld.lld", "-m", emulation, "-o", outpath, defpath) + return link("ld.lld", []string{"-m", emulation, "-o", outpath, defpath}) }, } jobs = append(jobs, job) diff --git a/builder/tools.go b/builder/tools.go index 33a26ec37e..79577e231f 100644 --- a/builder/tools.go +++ b/builder/tools.go @@ -56,7 +56,7 @@ func runCCompiler(flags ...string) error { } // link invokes a linker with the given name and flags. -func link(linker string, flags ...string) error { +func link(linker string, flags []string, env ...string) error { // We only support LLD. if linker != "ld.lld" && linker != "wasm-ld" { return fmt.Errorf("unexpected: linker %s should be ld.lld or wasm-ld", linker) @@ -75,6 +75,7 @@ func link(linker string, flags ...string) error { var buf bytes.Buffer cmd.Stdout = os.Stdout cmd.Stderr = &buf + cmd.Env = append(os.Environ(), env...) err := cmd.Run() if err != nil { if buf.Len() == 0 { From 4cb23977a0061d0837b1c8c05f424e38b481f558 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:39:19 -0700 Subject: [PATCH 4/6] builder: omit C command lines from trimmed debug info Some Clang builds record compiler arguments in debug information by default. Those arguments contain source, cache, and temporary paths that prefix maps do not rewrite. Even when the final binary omits that debug information, the paths affect ThinLTO symbol hashes. Disable command-line recording for trimmed C sources and libraries. The LLVM 18.1.8 Darwin regression reproduces this with independent builds. --- builder/library.go | 1 + compileopts/config.go | 1 + 2 files changed, 2 insertions(+) diff --git a/builder/library.go b/builder/library.go index 8a76950499..d98cdeacca 100644 --- a/builder/library.go +++ b/builder/library.go @@ -142,6 +142,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ "-ffile-prefix-map="+goenv.Get("TINYGOROOT")+"="+config.CSourcePath("github.com/tinygo-org/tinygo"), "-ffile-prefix-map="+goenv.Get("GOCACHE")+"="+config.CSourcePath("tinygo-cache"), "-fdebug-compilation-dir=.", + "-gno-record-command-line", ) } resourceDir := goenv.ClangResourceDir(false) diff --git a/compileopts/config.go b/compileopts/config.go index b7cf0d5ae7..f983248b41 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -417,6 +417,7 @@ func (c *Config) CFlags(libclang bool) []string { "-ffile-prefix-map="+goenv.Get("TINYGOROOT")+"="+c.CSourcePath("github.com/tinygo-org/tinygo"), "-ffile-prefix-map="+goenv.Get("GOCACHE")+"="+c.CSourcePath("tinygo-cache"), "-fdebug-compilation-dir=.", + "-gno-record-command-line", ) } // Always emit debug information. It is optionally stripped at link time. From 415cb129af693bbe4f790559e8b823fbfcac8406 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:45:39 -0700 Subject: [PATCH 5/6] builder: trim Clang builtin header paths Compiler-provided headers can live outside the TinyGo source tree when using a system LLVM installation. Their absolute paths remain in DWARF, as seen with __stddef_size_t.h in WASI test binaries on LLVM 18 and 19. Use the known Clang resource directory for trimmed builds so its headers can be recorded under a stable clang path. Apply this to both package compilation and cached libraries. --- builder/library.go | 5 ++++- compileopts/config.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/builder/library.go b/builder/library.go index d98cdeacca..d28bee24c9 100644 --- a/builder/library.go +++ b/builder/library.go @@ -137,6 +137,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ // reproducible. Otherwise the temporary directory is stored in the archive // itself, which varies each run. args := append(l.cflags(target, headerPath), "-c", "-Oz", "-gdwarf-4", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+compileopts.ClangTriple(target), "-fdebug-prefix-map="+dir+"="+remapDir) + resourceDir := goenv.ClangResourceDir(config.TrimPath()) if config.TrimPath() { args = append(args, "-ffile-prefix-map="+goenv.Get("TINYGOROOT")+"="+config.CSourcePath("github.com/tinygo-org/tinygo"), @@ -144,8 +145,10 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ "-fdebug-compilation-dir=.", "-gno-record-command-line", ) + if resourceDir != "" { + args = append(args, "-ffile-prefix-map="+resourceDir+"="+config.CSourcePath("clang")) + } } - resourceDir := goenv.ClangResourceDir(false) if resourceDir != "" { args = append(args, "-resource-dir="+resourceDir) } diff --git a/compileopts/config.go b/compileopts/config.go index f983248b41..a8b8d30274 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -396,7 +396,7 @@ func (c *Config) CFlags(libclang bool) []string { for _, flag := range c.Target.CFlags { cflags = append(cflags, strings.ReplaceAll(flag, "{root}", goenv.Get("TINYGOROOT"))) } - resourceDir := goenv.ClangResourceDir(libclang) + resourceDir := goenv.ClangResourceDir(libclang || c.TrimPath()) if resourceDir != "" { // The resource directory contains the built-in clang headers like // stdbool.h, stdint.h, float.h, etc. @@ -419,6 +419,9 @@ func (c *Config) CFlags(libclang bool) []string { "-fdebug-compilation-dir=.", "-gno-record-command-line", ) + if resourceDir != "" { + cflags = append(cflags, "-ffile-prefix-map="+resourceDir+"="+c.CSourcePath("clang")) + } } // Always emit debug information. It is optionally stripped at link time. cflags = append(cflags, "-gdwarf-4") From 6bf41a838dba99e296d0ea7ba214a689c13fd46d Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:20:05 -0700 Subject: [PATCH 6/6] loader: precompute recorded source directories Trimmed path lookups repeatedly resolve every package and module root. This repeats filesystem work for synthetic GOROOT paths and visits the same module once per package. Build immutable directory mappings after loading the package graph and deduplicate shared roots. Keep package mappings ahead of module mappings, with the same longest-directory and lexical tie-breaking rules. Add lookup benchmarks for 10, 100, and 1000 packages and regression coverage for mapping precedence and path boundaries. --- loader/loader.go | 55 +++++++++++++++++++++++--- loader/loader_test.go | 91 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 6 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index b21099d8d5..e257e05cf8 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -18,6 +18,7 @@ import ( "path" "path/filepath" "runtime" + "slices" "strconv" "strings" "unicode" @@ -39,6 +40,9 @@ type Program struct { sorted []*Package fset *token.FileSet + packagePaths []recordedPathMapping + modulePaths []recordedPathMapping + // Information obtained during parsing. LDFlags []string } @@ -262,6 +266,7 @@ func Load(config *compileopts.Config, inputPkg string, typeChecker types.Config) return p, NoTestFilesError{p.sorted[len(p.sorted)-1].ImportPath} } + p.initRecordedPaths() return p, nil } @@ -431,6 +436,38 @@ func (p *Package) sourceImportPath() string { return p.ImportPath } +type recordedPathMapping struct { + original string + recorded string +} + +func (p *Program) initRecordedPaths() { + if !p.config.TrimPath() { + return + } + packages := make(map[recordedPathMapping]struct{}) + modules := make(map[recordedPathMapping]struct{}) + for _, pkg := range p.Packages { + packages[recordedPathMapping{pkg.OriginalDir(), pkg.RecordedDir()}] = struct{}{} + modules[recordedPathMapping{pkg.OriginalModuleDir(), pkg.RecordedModuleDir()}] = struct{}{} + } + sorted := func(paths map[recordedPathMapping]struct{}) []recordedPathMapping { + result := make([]recordedPathMapping, 0, len(paths)) + for mapping := range paths { + result = append(result, mapping) + } + slices.SortFunc(result, func(a, b recordedPathMapping) int { + if len(a.original) != len(b.original) { + return len(b.original) - len(a.original) + } + return strings.Compare(a.recorded, b.recorded) + }) + return result + } + p.packagePaths = sorted(packages) + p.modulePaths = sorted(modules) +} + // RecordedPath returns the source path to record in the output. func (p *Package) RecordedPath(filename string) string { if !p.program.config.TrimPath() { @@ -459,13 +496,16 @@ func (p *Package) RecordedPath(filename string) string { var recordedPath string var matchedDirLength int - for _, sourcePkg := range p.program.Packages { - dir := sourcePkg.OriginalDir() + for _, mapping := range p.program.packagePaths { + dir := mapping.original + if len(dir) < matchedDirLength { + break + } rel, err := filepath.Rel(dir, filename) if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { continue } - candidate := path.Join(sourcePkg.RecordedDir(), filepath.ToSlash(rel)) + candidate := path.Join(mapping.recorded, filepath.ToSlash(rel)) if len(dir) > matchedDirLength || len(dir) == matchedDirLength && candidate < recordedPath { matchedDirLength = len(dir) recordedPath = candidate @@ -474,13 +514,16 @@ func (p *Package) RecordedPath(filename string) string { if recordedPath != "" { return recordedPath } - for _, sourcePkg := range p.program.Packages { - dir := sourcePkg.OriginalModuleDir() + for _, mapping := range p.program.modulePaths { + dir := mapping.original + if len(dir) < matchedDirLength { + break + } rel, err := filepath.Rel(dir, filename) if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { continue } - candidate := path.Join(sourcePkg.RecordedModuleDir(), filepath.ToSlash(rel)) + candidate := path.Join(mapping.recorded, filepath.ToSlash(rel)) if len(dir) > matchedDirLength || len(dir) == matchedDirLength && candidate < recordedPath { matchedDirLength = len(dir) recordedPath = candidate diff --git a/loader/loader_test.go b/loader/loader_test.go index 4ec5a04c54..1ea72bb64a 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -1,6 +1,7 @@ package loader import ( + "fmt" "path/filepath" "slices" "testing" @@ -9,6 +10,93 @@ import ( "github.com/tinygo-org/tinygo/goenv" ) +func BenchmarkRecordedPath(b *testing.B) { + for _, count := range []int{10, 100, 1000} { + b.Run(fmt.Sprintf("packages=%d", count), func(b *testing.B) { + program := &Program{ + config: &compileopts.Config{ + Options: &compileopts.Options{TrimPath: true}, + }, + goroot: filepath.FromSlash("/goroot"), + Packages: make(map[string]*Package), + } + var pkg *Package + for i := 0; i < count; i++ { + pkg = &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash(fmt.Sprintf("/tmp/module/pkg%d", i)), + ImportPath: fmt.Sprintf("example.com/module/pkg%d", i), + }, + } + pkg.Module.Path = "example.com/module" + pkg.Module.Dir = filepath.FromSlash("/tmp/module") + program.Packages[pkg.ImportPath] = pkg + } + program.initRecordedPaths() + for _, name := range []string{"package", "module"} { + b.Run(name, func(b *testing.B) { + filename := filepath.Join(pkg.Dir, "file.go") + want := pkg.ImportPath + "/file.go" + if name == "module" { + filename = filepath.FromSlash("/tmp/module/include/shared.h") + want = "example.com/module/include/shared.h" + } + b.ReportAllocs() + for b.Loop() { + if got := pkg.RecordedPath(filename); got != want { + b.Fatalf("RecordedPath() = %q, want %q", got, want) + } + } + }) + } + }) + } +} + +func TestRecordedPathPrecedence(t *testing.T) { + program := &Program{ + config: &compileopts.Config{ + Options: &compileopts.Options{TrimPath: true}, + }, + goroot: filepath.FromSlash("/goroot"), + Packages: make(map[string]*Package), + } + var pkg *Package + for _, entry := range []struct{ dir, name, moduleDir, modulePath string }{ + {"/tmp/root/pkg", "example.com/z", "/tmp/root", "example.com/root"}, + {"/tmp/root/pkg", "example.com/a", "/tmp/root", "example.com/root"}, + {"/tmp/root/pkg/nested", "example.com/nested", "/tmp/root", "example.com/root"}, + {"/tmp/root/pkg/include/sub", "example.com/includes/sub", "/tmp/root/pkg/include", "example.com/includes"}, + {"/tmp/root/extra/sub", "example.com/extra/sub", "/tmp/root/extra", "example.com/extra"}, + } { + pkg = &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash(entry.dir), + ImportPath: entry.name, + }, + } + pkg.Module.Dir = filepath.FromSlash(entry.moduleDir) + pkg.Module.Path = entry.modulePath + program.Packages[pkg.ImportPath] = pkg + } + program.initRecordedPaths() + for _, tc := range []struct{ filename, want string }{ + {"/tmp/root/pkg/file.go", "example.com/a/file.go"}, + {"/tmp/root/pkg/nested/file.go", "example.com/nested/file.go"}, + {"/tmp/root/pkg/include/shared.h", "example.com/a/include/shared.h"}, + {"/tmp/root/extra/include/shared.h", "example.com/extra/include/shared.h"}, + {"/tmp/root/include/shared.h", "example.com/root/include/shared.h"}, + {"/tmp/root/pkg-other/file.go", "example.com/root/pkg-other/file.go"}, + {"/tmp/root-other/file.go", filepath.FromSlash("/tmp/root-other/file.go")}, + } { + if got := pkg.RecordedPath(filepath.FromSlash(tc.filename)); got != tc.want { + t.Errorf("RecordedPath(%q) = %q, want %q", tc.filename, got, tc.want) + } + } +} + func TestRecordedPath(t *testing.T) { config := &compileopts.Config{ Options: &compileopts.Options{TrimPath: true}, @@ -48,6 +136,7 @@ func TestRecordedPath(t *testing.T) { } program.Packages[pkg.ImportPath] = pkg program.Packages[dependency.ImportPath] = dependency + program.initRecordedPaths() if got, want := dependency.RecordedDir(), "example.com/dependency@v1.2.3/subpackage"; got != want { t.Fatalf("RecordedDir() = %q, want %q", got, want) @@ -98,6 +187,7 @@ func TestRecordedPath(t *testing.T) { vendored.Module.Path = "example.com/dependency" vendored.Module.Version = "v1.2.3" program.Packages[vendored.ImportPath] = vendored + program.initRecordedPaths() if got, want := vendored.OriginalModuleDir(), filepath.FromSlash("/tmp/main/vendor/example.com/dependency"); got != want { t.Fatalf("vendored OriginalModuleDir() = %q, want %q", got, want) } @@ -118,6 +208,7 @@ func TestRecordedPath(t *testing.T) { }, } program.Packages[gopath.ImportPath] = gopath + program.initRecordedPaths() if got, want := gopath.OriginalModuleDir(), filepath.FromSlash("/gopath/src"); got != want { t.Fatalf("GOPATH OriginalModuleDir() = %q, want %q", got, want) }