diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml index cc53fae05f..ea7ef41e3a 100644 --- a/.github/workflows/compat.yml +++ b/.github/workflows/compat.yml @@ -69,4 +69,5 @@ jobs: - run: tinygo version - run: make gen-device -j4 - run: go test -tags=llvm${{ env.LLVM }} -short -skip=TestErrors + - run: go test -tags=llvm${{ env.LLVM }} -run TestTrimPath -target=darwin/amd64 - run: make smoketest-quick XTENSA=0 diff --git a/builder/build.go b/builder/build.go index dab5a590c2..d41062c530 100644 --- a/builder/build.go +++ b/builder/build.go @@ -12,8 +12,10 @@ import ( "encoding/json" "errors" "fmt" + "go/token" "go/types" "hash/crc32" + "io" "maps" "math/bits" "os" @@ -30,6 +32,7 @@ import ( "github.com/gofrs/flock" "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/compiler" + "github.com/tinygo-org/tinygo/compiler/llvmutil" "github.com/tinygo-org/tinygo/goenv" "github.com/tinygo-org/tinygo/interp" "github.com/tinygo-org/tinygo/loader" @@ -37,7 +40,6 @@ import ( "github.com/tinygo-org/tinygo/transform" "golang.org/x/mod/module" "golang.org/x/mod/semver" - "golang.org/x/tools/go/ssa" "tinygo.org/x/go-llvm" ) @@ -132,6 +134,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe }, "testing": {}, } + if config.TrimPath() { + globalValues["runtime"]["goroot"] = "" + } if config.TestConfig.CompileTestBinary { // The testing.testBinary is set to "1" when in a test. // This is needed for testing.Testing() to work correctly. @@ -213,6 +218,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe RelocationModel: config.RelocationModel(), SizeLevel: sizeLevel, TinyGoVersion: goenv.Version(), + TrimPath: config.TrimPath(), Scheduler: config.Scheduler(), AutomaticStackSize: config.AutomaticStackSize(), @@ -279,6 +285,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe result.PackagePathMap = make(map[string]string, len(lprogram.Packages)) for _, pkg := range lprogram.Sorted() { result.PackagePathMap[pkg.OriginalDir()] = pkg.Pkg.Path() + if config.TrimPath() { + result.PackagePathMap[filepath.FromSlash(pkg.RecordedDir())] = pkg.Pkg.Path() + } } // Strip default initializers for -X globals from the type info before @@ -349,7 +358,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe } } - job.result, err = createEmbedObjectFile(string(data), hexSum, name, pkg.OriginalDir(), tmpdir, compilerConfig) + job.result, err = createEmbedObjectFile(string(data), hexSum, name, pkg.RecordedDir(), tmpdir, compilerConfig) return err }, } @@ -383,7 +392,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe CompilerBuildID: string(compilerBuildID), LLVMVersion: llvm.Version, Config: compilerConfig, - CFlags: pkg.CFlags, + CFlags: pkg.RecordedCFlags(), FileHashes: make(map[string]string, len(pkg.FileHashes)), EmbeddedFiles: make(map[string]string, len(allFiles)), Imports: make(map[string]string, len(pkg.Pkg.Imports())), @@ -391,7 +400,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe UndefinedGlobals: undefinedGlobals, } for filePath, hash := range pkg.FileHashes { - actionID.FileHashes[filePath] = hex.EncodeToString(hash) + actionID.FileHashes[pkg.RecordedPath(filePath)] = hex.EncodeToString(hash) } for name, files := range allFiles { actionID.EmbeddedFiles[name] = files[0].Hash @@ -450,9 +459,10 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // These headers could be compiled in parallel but the benefit // is so small that it's probably not worth parallelizing. // Packages are compiled independently anyway. - for _, cgoHeader := range pkg.CGoHeaders { + packageNameHash := sha256.Sum256([]byte(pkg.ImportPath)) + for i, cgoHeader := range pkg.CGoHeaders { // Store the header text in a temporary file. - f, err := os.CreateTemp(tmpdir, "cgosnippet-*.c") + f, err := os.Create(filepath.Join(tmpdir, fmt.Sprintf("cgosnippet-%x-%d.c", packageNameHash, i))) if err != nil { return err } @@ -464,6 +474,12 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // Compile the code (if there is any) to bitcode. flags := append([]string{"-c", "-emit-llvm", "-o", f.Name() + ".bc", f.Name()}, pkg.CFlags...) + flags = append(flags, "-working-directory="+tmpdir) + if config.TrimPath() { + flags = append(flags, + "-ffile-prefix-map="+tmpdir+"="+config.CSourcePath(pkg.RecordedDir()), + ) + } if config.Options.PrintCommands != nil { config.Options.PrintCommands("clang", flags...) } @@ -633,7 +649,9 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe defer irbuilder.Dispose() irbuilder.SetInsertPointAtEnd(block) if config.Debug() && !config.Options.SkipDWARF { - addInitAllDebugInfo(mod, llvmInitFn, irbuilder, program) + pos := program.Fset.Position(program.ImportedPackage("runtime").Members["initAll"].Pos()) + pos.Filename = lprogram.Packages["runtime"].RecordedPath(pos.Filename) + addInitAllDebugInfo(mod, llvmInitFn, irbuilder, pos, config.TrimPath()) } ptrType := llvm.PointerType(mod.Context().Int8Type(), 0) for _, pkg := range lprogram.Sorted() { @@ -802,7 +820,13 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe job := &compileJob{ description: "compile extra file " + path, run: func(job *compileJob) error { - result, err := compileAndCacheCFile(abspath, tmpdir, config.CFlags(false), config.Options.PrintCommands) + var compileConfig *cFileCompileConfig + if config.TrimPath() { + compileConfig = &cFileCompileConfig{ + recordedPath: config.CSourcePath(filepath.Join("github.com/tinygo-org/tinygo", path)), + } + } + result, err := compileAndCacheCFile(abspath, tmpdir, config.CFlags(false), compileConfig, config.Options.PrintCommands) job.result = result return err }, @@ -819,7 +843,14 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe job := &compileJob{ description: "compile CGo file " + abspath, run: func(job *compileJob) error { - result, err := compileAndCacheCFile(abspath, tmpdir, pkg.CFlags, config.Options.PrintCommands) + cflags := pkg.CFlags + var compileConfig *cFileCompileConfig + if config.TrimPath() { + compileConfig = &cFileCompileConfig{ + recordedPath: config.CSourcePath(pkg.RecordedPath(abspath)), + } + } + result, err := compileAndCacheCFile(abspath, tmpdir, cflags, compileConfig, config.Options.PrintCommands) job.result = result return err }, @@ -872,11 +903,18 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe description: "link", dependencies: linkerDependencies, run: func(job *compileJob) error { - for _, dependency := range job.dependencies { + for i, dependency := range job.dependencies { if dependency.result == "" { return errors.New("dependency without result: " + dependency.description) } - ldflags = append(ldflags, dependency.result) + linkerInput := dependency.result + if config.TrimPath() && config.LinkerFlavor() == "darwin" { + linkerInput = filepath.Join(tmpdir, fmt.Sprintf("link-input-%d%s", i, filepath.Ext(linkerInput))) + if err := linkOrCopyFile(dependency.result, linkerInput); err != nil { + return err + } + } + ldflags = append(ldflags, linkerInput) } ldflags = append(ldflags, "-mllvm", "-mcpu="+config.CPU()) ldflags = append(ldflags, "-mllvm", "-mattr="+config.Features()) // needed for MIPS softfloat @@ -888,9 +926,15 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe "--thinlto-cache-dir="+filepath.Join(cacheDir, "thinlto")) case "darwin": // Options for the ld64-compatible lld linker. - ldflags = append(ldflags, - "--lto-O"+strconv.Itoa(speedLevel), - "-cache_path_lto", filepath.Join(cacheDir, "thinlto")) + ldflags = append(ldflags, "--lto-O"+strconv.Itoa(speedLevel)) + // LLD 15 embeds cache paths in OSO symbols, unlike LLD 16+. + // Fixed in LLD 16 by https://reviews.llvm.org/D131624. + if !config.TrimPath() || llvmutil.Version() >= 16 { + ldflags = append(ldflags, "-cache_path_lto", filepath.Join(cacheDir, "thinlto")) + } + if config.TrimPath() { + ldflags = append(ldflags, "-oso_prefix", tmpdir+string(filepath.Separator)) + } case "gnu": // Options for the ELF linker. ldflags = append(ldflags, @@ -914,7 +958,11 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe if config.Options.PrintCommands != nil { config.Options.PrintCommands(config.Target.Linker, ldflags...) } - err = link(config.Target.Linker, ldflags...) + var linkEnv []string + if config.TrimPath() && config.LinkerFlavor() == "darwin" { + linkEnv = append(linkEnv, "ZERO_AR_DATE=1") + } + err = link(config.Target.Linker, ldflags, linkEnv...) if err != nil { return err } @@ -1139,6 +1187,34 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe return result, nil } +func linkOrCopyFile(src, dst string) error { + if err := os.Link(src, dst); err == nil { + return nil + } + + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return err + } + _, copyErr := io.Copy(destination, source) + closeErr := destination.Close() + if copyErr != nil { + os.Remove(dst) + return copyErr + } + if closeErr != nil { + os.Remove(dst) + return closeErr + } + return nil +} + // createEmbedObjectFile creates a new object file with the given contents, for // the embed package. func createEmbedObjectFile(data, hexSum, sourceFile, sourceDir, tmpdir string, compilerConfig *compiler.Config) (string, error) { @@ -1228,7 +1304,8 @@ func createEmbedObjectFile(data, hexSum, sourceFile, sourceDir, tmpdir string, c return "", err } defer machine.Dispose() - outfile, err := os.CreateTemp(tmpdir, "embed-"+hexSum+"-*.o") + sourcePathHash := sha256.Sum256([]byte(filepath.ToSlash(filepath.Join(sourceDir, sourceFile)))) + outfile, err := os.Create(filepath.Join(tmpdir, "embed-"+hexSum+"-"+hex.EncodeToString(sourcePathHash[:8])+".o")) if err != nil { return "", err } @@ -1281,15 +1358,18 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error { // addInitAllDebugInfo gives runtime.initAll a subprogram so that code emitted // into it by interp or the inliner keeps its line information. -func addInitAllDebugInfo(mod llvm.Module, fn llvm.Value, irbuilder llvm.Builder, program *ssa.Program) { - pos := program.Fset.Position(program.ImportedPackage("runtime").Members["initAll"].Pos()) +func addInitAllDebugInfo(mod llvm.Module, fn llvm.Value, irbuilder llvm.Builder, pos token.Position, trimPath bool) { dir, file := filepath.Split(pos.Filename) + compileDir, compileFile := filepath.Clean(dir), file + if trimPath { + compileDir, compileFile = "", pos.Filename + } dibuilder := llvm.NewDIBuilder(mod) defer dibuilder.Destroy() dibuilder.CreateCompileUnit(llvm.DICompileUnit{ Language: 0xb, // DW_LANG_C99 (0xc, off-by-one?) - File: file, - Dir: filepath.Clean(dir), + File: compileFile, + Dir: compileDir, Producer: "TinyGo", Optimized: true, }) diff --git a/builder/cc.go b/builder/cc.go index 9cc03790b6..6bf3a99225 100644 --- a/builder/cc.go +++ b/builder/cc.go @@ -12,6 +12,7 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "sort" "strings" @@ -21,6 +22,10 @@ import ( "tinygo.org/x/go-llvm" ) +type cFileCompileConfig struct { + recordedPath string +} + // compileAndCacheCFile compiles a C or assembly file using a build cache. // Compiling the same file again (if nothing changed, including included header // files) the output is loaded from the build cache instead. @@ -56,7 +61,7 @@ import ( // depfile but without invalidating its name. For this reason, the depfile is // written on each new compilation (even when it seems unnecessary). However, it // could in rare cases lead to a stale file fetched from the cache. -func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands func(string, ...string)) (string, error) { +func compileAndCacheCFile(abspath, tmpdir string, cflags []string, compileConfig *cFileCompileConfig, printCommands func(string, ...string)) (string, error) { // Hash input file. fileHash, err := hashFile(abspath) if err != nil { @@ -68,16 +73,22 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands defer unlock() // Create cache key for the dependencies file. + recordedPath := "" + if compileConfig != nil { + recordedPath = compileConfig.recordedPath + } buf, err := json.Marshal(struct { - Path string - Hash string - Flags []string - LLVMVersion string + Path string + Hash string + Flags []string + RecordedPath string + LLVMVersion string }{ - Path: abspath, - Hash: fileHash, - Flags: cflags, - LLVMVersion: llvm.Version, + Path: abspath, + Hash: fileHash, + Flags: cflags, + RecordedPath: recordedPath, + LLVMVersion: llvm.Version, }) if err != nil { panic(err) // shouldn't happen @@ -124,7 +135,17 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands depTmpFile.Close() flags := append([]string{}, cflags...) // copy cflags flags = append(flags, "-MD", "-MV", "-MTdeps", "-MF", depTmpFile.Name(), "-flto=thin") // autogenerate dependencies - flags = append(flags, "-c", "-o", objTmpFile.Name(), abspath) + // Match cmd/go/internal/work/exec.go's ccompile directory in both modes. + workingDir := filepath.Dir(abspath) + flags = append(flags, "-working-directory="+workingDir) + if compileConfig != nil { + seed := sha512.Sum512_224([]byte(compileConfig.recordedPath)) + flags = append(flags, + "-fdebug-compilation-dir="+path.Dir(compileConfig.recordedPath), + "-frandom-seed="+hex.EncodeToString(seed[:]), + ) + } + flags = append(flags, "-c", "-o", objTmpFile.Name(), filepath.Base(abspath)) if strings.ToLower(filepath.Ext(abspath)) == ".s" { // If this is an assembly file (.s or .S, lowercase or uppercase), then // we'll need to add -Qunused-arguments because many parameters are @@ -145,6 +166,11 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands if err != nil { return "", err } + for i, dependencyPath := range dependencyPaths { + if !filepath.IsAbs(dependencyPath) { + dependencyPaths[i] = filepath.Join(workingDir, dependencyPath) + } + } dependencyPaths = append(dependencyPaths, abspath) // necessary for .s files dependencySet := make(map[string]struct{}, len(dependencyPaths)) var dependencySlice []string diff --git a/builder/darwin-libsystem.go b/builder/darwin-libsystem.go index e47b54230e..de927e2eb8 100644 --- a/builder/darwin-libsystem.go +++ b/builder/darwin-libsystem.go @@ -52,7 +52,7 @@ func makeDarwinLibSystemJob(config *compileopts.Config, tmpdir string) *compileJ if config.Options.PrintCommands != nil { config.Options.PrintCommands("ld.lld", flags...) } - return link("ld.lld", flags...) + return link("ld.lld", flags) }, } } diff --git a/builder/library.go b/builder/library.go index 722f3c8371..d28bee24c9 100644 --- a/builder/library.go +++ b/builder/library.go @@ -123,6 +123,9 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ } remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name) + if config.TrimPath() { + remapDir = config.CSourcePath(filepath.Join("github.com/tinygo-org/tinygo", ".tinygo-build", l.name)) + } dir := filepath.Join(tmpdir, "build-lib-"+l.name) err = os.Mkdir(dir, 0777) if err != nil { @@ -134,7 +137,18 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ // reproducible. Otherwise the temporary directory is stored in the archive // itself, which varies each run. args := append(l.cflags(target, headerPath), "-c", "-Oz", "-gdwarf-4", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+compileopts.ClangTriple(target), "-fdebug-prefix-map="+dir+"="+remapDir) - resourceDir := goenv.ClangResourceDir(false) + resourceDir := goenv.ClangResourceDir(config.TrimPath()) + if config.TrimPath() { + args = append(args, + "-ffile-prefix-map="+goenv.Get("TINYGOROOT")+"="+config.CSourcePath("github.com/tinygo-org/tinygo"), + "-ffile-prefix-map="+goenv.Get("GOCACHE")+"="+config.CSourcePath("tinygo-cache"), + "-fdebug-compilation-dir=.", + "-gno-record-command-line", + ) + if resourceDir != "" { + args = append(args, "-ffile-prefix-map="+resourceDir+"="+config.CSourcePath("clang")) + } + } if resourceDir != "" { args = append(args, "-resource-dir="+resourceDir) } diff --git a/builder/mingw-w64.go b/builder/mingw-w64.go index 64c3be3f4c..d0beba9e80 100644 --- a/builder/mingw-w64.go +++ b/builder/mingw-w64.go @@ -165,7 +165,7 @@ func makeMinGWExtraLibs(tmpdir, goarch string) []*compileJob { return err } } - return link("ld.lld", "-m", emulation, "-o", outpath, defpath) + return link("ld.lld", []string{"-m", emulation, "-o", outpath, defpath}) }, } jobs = append(jobs, job) diff --git a/builder/sizes.go b/builder/sizes.go index 98a830d9d2..1f8e639c31 100644 --- a/builder/sizes.go +++ b/builder/sizes.go @@ -17,6 +17,7 @@ import ( "strings" "github.com/aykevl/go-wasm" + "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/goenv" ) @@ -946,6 +947,17 @@ func readSection(section memorySection, addresses []addressLine, program *progra // findPackagePath returns the Go package (or a pseudo package) for the given // path. It uses some heuristics, for example for some C libraries. func findPackagePath(path string, packagePathMap map[string]string) (packagePath, filename string) { + for _, prefix := range []string{compileopts.CSourcePathRoot + "/", compileopts.CSourcePathRootWindows + "/"} { + if strings.HasPrefix(filepath.ToSlash(path), prefix) { + recordedPath := strings.TrimPrefix(filepath.ToSlash(path), prefix) + if rel, ok := strings.CutPrefix(recordedPath, "github.com/tinygo-org/tinygo/"); ok { + path = filepath.Join(goenv.Get("TINYGOROOT"), filepath.FromSlash(rel)) + } else { + path = filepath.FromSlash(recordedPath) + } + break + } + } // Check whether this path is part of one of the compiled packages. packagePath, ok := packagePathMap[filepath.Dir(path)] if ok { diff --git a/builder/sizes_test.go b/builder/sizes_test.go index 270a8a3afc..c1e6cabce7 100644 --- a/builder/sizes_test.go +++ b/builder/sizes_test.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "regexp" "runtime" "strconv" @@ -12,6 +13,7 @@ import ( "time" "github.com/tinygo-org/tinygo/compileopts" + "github.com/tinygo-org/tinygo/goenv" ) var sema = make(chan struct{}, runtime.NumCPU()) @@ -24,6 +26,29 @@ type sizeTest struct { opt string } +func TestTrimPathPackageSize(t *testing.T) { + root := goenv.Get("TINYGOROOT") + packages := map[string]string{ + filepath.FromSlash("example.com/main"): "main", + filepath.FromSlash("example.com/dependency@v1.2.3/subpackage"): "example.com/dependency/subpackage", + filepath.Join(root, "src", "runtime"): "runtime", + } + for _, tc := range []struct{ path, pkg, file string }{ + {"example.com/main/main.go", "main", "main.go"}, + {"/_/example.com/main/main.c", "main", "main.c"}, + {"//_/_/example.com/main/main.c", "main", "main.c"}, + {"/_/example.com/dependency@v1.2.3/subpackage/dependency.c", "example.com/dependency/subpackage", "dependency.c"}, + {"/_/github.com/tinygo-org/tinygo/src/runtime/runtime_unix.c", "runtime", "runtime_unix.c"}, + {"/_/github.com/tinygo-org/tinygo/lib/musl/src/stdio/puts.c", "C musl", filepath.FromSlash("src/stdio/puts.c")}, + {"//_/_/github.com/tinygo-org/tinygo/lib/bdwgc/alloc.c", "C bdwgc", "alloc.c"}, + } { + pkg, file := findPackagePath(filepath.FromSlash(tc.path), packages) + if pkg != tc.pkg || file != tc.file { + t.Errorf("findPackagePath(%q) = %q, %q; want %q, %q", tc.path, pkg, file, tc.pkg, tc.file) + } + } +} + // Test whether code and data size is as expected for the given targets. // This tests both the logic of loadProgramSize and checks that code size // doesn't change unintentionally. @@ -116,7 +141,7 @@ func measureBinarySizes(t *testing.T, tests []sizeTest) []*programSize { } func measureBinarySize(tc sizeTest, tmpdir string) (*programSize, error) { - result, err := buildBinaryInDir(tc.target, tc.path, tc.opt, tmpdir) + result, err := buildBinaryInDir(tc.target, tc.path, tc.opt, tmpdir, false) if err != nil { return nil, err } @@ -162,20 +187,25 @@ func formatSizeTable(tests []sizeTest, sizes []*programSize) string { // Check that the -size=full flag attributes binary size to the correct package // without filesystem paths and things like that. func TestSizeFull(t *testing.T) { - tests := []string{ - "microbit", - "wasip1", + tests := []struct { + target string + trimPath bool + }{ + {"microbit", false}, + {"microbit", true}, + {"wasip1", false}, + {"wasip1", true}, } libMatch := regexp.MustCompile(`^C [a-z -]+$`) // example: "C interrupt vector" pkgMatch := regexp.MustCompile(`^[a-z/]+$`) // example: "internal/task" - for _, target := range tests { - t.Run(target, func(t *testing.T) { + for _, test := range tests { + t.Run(fmt.Sprintf("%s/trimpath=%v", test.target, test.trimPath), func(t *testing.T) { t.Parallel() // Build the binary. - result := buildBinary(t, target, "examples/serial") + result := buildBinary(t, test.target, "examples/serial", test.trimPath) // Check whether the binary doesn't contain any unexpected package // names. @@ -200,16 +230,16 @@ func TestSizeFull(t *testing.T) { } } -func buildBinary(t *testing.T, targetString, pkgName string) BuildResult { +func buildBinary(t *testing.T, targetString, pkgName string, trimPath bool) BuildResult { t.Helper() - result, err := buildBinaryInDir(targetString, pkgName, "z", t.TempDir()) + result, err := buildBinaryInDir(targetString, pkgName, "z", t.TempDir(), trimPath) if err != nil { t.Fatal(err) } return result } -func buildBinaryInDir(targetString, pkgName, opt, tmpdir string) (BuildResult, error) { +func buildBinaryInDir(targetString, pkgName, opt, tmpdir string, trimPath bool) (BuildResult, error) { options := compileopts.Options{ Target: targetString, Opt: opt, @@ -217,6 +247,7 @@ func buildBinaryInDir(targetString, pkgName, opt, tmpdir string) (BuildResult, e InterpTimeout: 60 * time.Second, Debug: true, VerifyIR: true, + TrimPath: trimPath, } target, err := compileopts.LoadTarget(&options) if err != nil { diff --git a/builder/tools.go b/builder/tools.go index 33a26ec37e..79577e231f 100644 --- a/builder/tools.go +++ b/builder/tools.go @@ -56,7 +56,7 @@ func runCCompiler(flags ...string) error { } // link invokes a linker with the given name and flags. -func link(linker string, flags ...string) error { +func link(linker string, flags []string, env ...string) error { // We only support LLD. if linker != "ld.lld" && linker != "wasm-ld" { return fmt.Errorf("unexpected: linker %s should be ld.lld or wasm-ld", linker) @@ -75,6 +75,7 @@ func link(linker string, flags ...string) error { var buf bytes.Buffer cmd.Stdout = os.Stdout cmd.Stderr = &buf + cmd.Env = append(os.Environ(), env...) err := cmd.Run() if err != nil { if buf.Len() == 0 { diff --git a/compileopts/cflags.go b/compileopts/cflags.go new file mode 100644 index 0000000000..3c620bc842 --- /dev/null +++ b/compileopts/cflags.go @@ -0,0 +1,44 @@ +package compileopts + +import "strings" + +// RewriteCFlags rewrites path operands without changing other compiler options. +func RewriteCFlags(flags []string, rewrite func(string) string) []string { + flags = append([]string(nil), flags...) + nextIsPath := false + for i, flag := range flags { + if nextIsPath { + flags[i] = rewrite(flag) + nextIsPath = false + continue + } + switch flag { + case "-I", "-L", "-F", "-isystem", "-isystem-after", "-iquote", "-idirafter", "-include", "--include", "-imacros", "--imacros", "-include-pch", "-isysroot", "--sysroot", "-resource-dir", "-iframework", "-iframeworkwithsysroot": + nextIsPath = true + continue + } + for _, prefix := range []string{ + "-I", "-L", "-F", + "-isystem-after", "-isystem", "-iquote", "-idirafter", + "-iframeworkwithsysroot", "-iframework", + "-include", "--include", "-imacros", "--imacros", + "--sysroot=", "-isysroot=", "-isysroot", "-resource-dir=", + "-fmodule-map-file=", "-fmodules-cache-path=", + } { + if strings.HasPrefix(flag, prefix) { + flags[i] = prefix + rewrite(strings.TrimPrefix(flag, prefix)) + break + } + } + for _, prefix := range []string{"-fdebug-prefix-map=", "-ffile-prefix-map=", "-fmacro-prefix-map="} { + if strings.HasPrefix(flag, prefix) { + mapping := strings.TrimPrefix(flag, prefix) + if sep := strings.LastIndexByte(mapping, '='); sep >= 0 { + flags[i] = prefix + rewrite(mapping[:sep]) + mapping[sep:] + } + break + } + } + } + return flags +} diff --git a/compileopts/config.go b/compileopts/config.go index 1426ac424c..a8b8d30274 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "path" "path/filepath" "regexp" "slices" @@ -59,6 +60,26 @@ func (c *Config) BuildMode() string { return "default" } +// TrimPath reports whether local file system paths must be removed from the +// output. +func (c *Config) TrimPath() bool { + return c.Options.TrimPath +} + +const ( + CSourcePathRoot = "/_" + CSourcePathRootWindows = "//_/_" +) + +// CSourcePath avoids relative DWARF paths being joined to DW_AT_comp_dir. +// See go.dev/src/cmd/go/internal/work/exec.go (Builder.ccompile). +func (c *Config) CSourcePath(filename string) string { + if c.GOOS() == "windows" { + return "/" + path.Join(CSourcePathRootWindows, filepath.ToSlash(filename)) + } + return path.Join(CSourcePathRoot, filepath.ToSlash(filename)) +} + // Features returns a list of features this CPU supports. For example, for a // RISC-V processor, that could be "+a,+c,+m". For many targets, an empty list // will be returned. @@ -336,6 +357,9 @@ func (c *Config) LibraryPath(name string) string { if c.LibcNeedsMalloc() { options += "+malloc" } + if c.TrimPath() { + options += "+trimpath-v1" + } // No precompiled library found. Determine the path name that will be used // in the build cache. @@ -372,7 +396,7 @@ func (c *Config) CFlags(libclang bool) []string { for _, flag := range c.Target.CFlags { cflags = append(cflags, strings.ReplaceAll(flag, "{root}", goenv.Get("TINYGOROOT"))) } - resourceDir := goenv.ClangResourceDir(libclang) + resourceDir := goenv.ClangResourceDir(libclang || c.TrimPath()) if resourceDir != "" { // The resource directory contains the built-in clang headers like // stdbool.h, stdint.h, float.h, etc. @@ -388,6 +412,17 @@ func (c *Config) CFlags(libclang bool) []string { "-I"+filepath.Join(goenv.Get("TINYGOROOT"), "lib", "bdwgc", "include"), ) } + if c.TrimPath() { + cflags = append(cflags, + "-ffile-prefix-map="+goenv.Get("TINYGOROOT")+"="+c.CSourcePath("github.com/tinygo-org/tinygo"), + "-ffile-prefix-map="+goenv.Get("GOCACHE")+"="+c.CSourcePath("tinygo-cache"), + "-fdebug-compilation-dir=.", + "-gno-record-command-line", + ) + if resourceDir != "" { + cflags = append(cflags, "-ffile-prefix-map="+resourceDir+"="+c.CSourcePath("clang")) + } + } // Always emit debug information. It is optionally stripped at link time. cflags = append(cflags, "-gdwarf-4") // Use the same optimization level as TinyGo. diff --git a/compileopts/options.go b/compileopts/options.go index 750dcb42dc..ad23c38471 100644 --- a/compileopts/options.go +++ b/compileopts/options.go @@ -31,6 +31,7 @@ type Options struct { Directory string // working dir, leave it unset to use the current working dir Target string BuildMode string // -buildmode flag + TrimPath bool // -trimpath flag Opt string GC string PanicStrategy string diff --git a/compiler/compiler.go b/compiler/compiler.go index e4bb613840..7914c0ade5 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -50,6 +50,7 @@ type Config struct { RelocationModel string SizeLevel int TinyGoVersion string // for llvm.ident + TrimPath bool // Various compiler options that determine how code is generated. Scheduler string @@ -861,7 +862,8 @@ func (c *compilerContext) attachDebugInfoRawWithDefinition(f *ssa.Function, llvm // one. func (c *compilerContext) getDIFile(filename string) llvm.Metadata { if _, ok := c.difiles[filename]; !ok { - dir, file := filepath.Split(filename) + recordedPath := c.loaderPkg.RecordedPath(filename) + dir, file := filepath.Split(recordedPath) if dir != "" { dir = dir[:len(dir)-1] } @@ -883,7 +885,7 @@ func (c *compilerContext) createPackage(irbuilder llvm.Builder, pkg *ssa.Package sort.Slice(members, func(i, j int) bool { iPos := pkg.Members[members[i]].Pos() jPos := pkg.Members[members[j]].Pos() - if i == j { + if iPos == jPos { // Cannot sort by pos, so do it by name. return members[i] < members[j] } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index 4831a6868e..d66b26b616 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -137,6 +137,25 @@ func TestCompiler(t *testing.T) { } } +func TestCGoGlobalOrder(t *testing.T) { + for range 10 { + mod, errs := testCompilePackage(t, &compileopts.Options{Target: "wasm"}, "cgo-global-order.go") + if len(errs) != 0 { + t.Fatal(errs) + } + var names []string + for global := mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if strings.HasSuffix(global.Name(), "$funcaddr") { + names = append(names, global.Name()) + } + } + mod.Dispose() + if len(names) != 4 || !slices.IsSorted(names) { + t.Fatalf("CGo globals are not in name order: %v", names) + } + } +} + func TestOptimizedLargeAggregateABI(t *testing.T) { options := &compileopts.Options{Target: "wasm"} mod, errs := testCompilePackage(t, options, "large-optimized.go") diff --git a/compiler/testdata/cgo-global-order.go b/compiler/testdata/cgo-global-order.go new file mode 100644 index 0000000000..b72251be45 --- /dev/null +++ b/compiler/testdata/cgo-global-order.go @@ -0,0 +1,16 @@ +package main + +/* +void d(void); +void c(void); +void b(void); +void a(void); +*/ +import "C" + +func main() { + C.d() + C.c() + C.b() + C.a() +} diff --git a/loader/loader.go b/loader/loader.go index df46d662fa..e257e05cf8 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -18,6 +18,7 @@ import ( "path" "path/filepath" "runtime" + "slices" "strconv" "strings" "unicode" @@ -39,6 +40,9 @@ type Program struct { sorted []*Package fset *token.FileSet + packagePaths []recordedPathMapping + modulePaths []recordedPathMapping + // Information obtained during parsing. LDFlags []string } @@ -262,6 +266,7 @@ func Load(config *compileopts.Config, inputPkg string, typeChecker types.Config) return p, NoTestFilesError{p.sorted[len(p.sorted)-1].ImportPath} } + p.initRecordedPaths() return p, nil } @@ -365,6 +370,206 @@ func (p *Package) OriginalDir() string { return strings.TrimSuffix(p.program.getOriginalPath(p.Dir+string(os.PathSeparator)), string(os.PathSeparator)) } +// OriginalModuleDir returns the real module directory. +func (p *Package) OriginalModuleDir() string { + if p.Module.Dir != "" { + return strings.TrimSuffix(p.program.getOriginalPath(p.Module.Dir+string(os.PathSeparator)), string(os.PathSeparator)) + } + if p.Module.Path == "" && p.Root != "" { + root := filepath.Join(p.Root, "src") + return strings.TrimSuffix(p.program.getOriginalPath(root+string(os.PathSeparator)), string(os.PathSeparator)) + } + if p.Module.Path != "" { + importPath := p.sourceImportPath() + rel := "." + if importPath != p.Module.Path { + rel = strings.TrimPrefix(importPath, p.Module.Path+"/") + } + if rel == "." || rel != importPath { + dir := p.OriginalDir() + if rel != "." { + for range strings.Split(rel, "/") { + dir = filepath.Dir(dir) + } + } + return dir + } + } + return p.OriginalDir() +} + +// RecordedModuleDir returns the module directory to record in the output. +func (p *Package) RecordedModuleDir() string { + if !p.program.config.TrimPath() { + return p.OriginalModuleDir() + } + if p.Module.Path == "" { + if p.Root != "" { + return "." + } + return p.RecordedDir() + } + if p.Module.Version != "" { + return p.Module.Path + "@" + p.Module.Version + } + return p.Module.Path +} + +// RecordedDir returns the source directory to record in the output. +func (p *Package) RecordedDir() string { + if !p.program.config.TrimPath() { + return p.OriginalDir() + } + if p.Module.Path != "" && p.Module.Version != "" { + return path.Join(p.Module.Path+"@"+p.Module.Version, strings.TrimPrefix(p.sourceImportPath(), p.Module.Path)) + } + return p.sourceImportPath() +} + +func (p *Package) sourceImportPath() string { + if p.ForTest != "" && p.ImportPath == p.ForTest+"_test" { + return p.ForTest + } + if p.program.config.TestConfig.CompileTestBinary && p == p.program.MainPkg() { + return strings.TrimSuffix(p.ImportPath, ".test") + } + return p.ImportPath +} + +type recordedPathMapping struct { + original string + recorded string +} + +func (p *Program) initRecordedPaths() { + if !p.config.TrimPath() { + return + } + packages := make(map[recordedPathMapping]struct{}) + modules := make(map[recordedPathMapping]struct{}) + for _, pkg := range p.Packages { + packages[recordedPathMapping{pkg.OriginalDir(), pkg.RecordedDir()}] = struct{}{} + modules[recordedPathMapping{pkg.OriginalModuleDir(), pkg.RecordedModuleDir()}] = struct{}{} + } + sorted := func(paths map[recordedPathMapping]struct{}) []recordedPathMapping { + result := make([]recordedPathMapping, 0, len(paths)) + for mapping := range paths { + result = append(result, mapping) + } + slices.SortFunc(result, func(a, b recordedPathMapping) int { + if len(a.original) != len(b.original) { + return len(b.original) - len(a.original) + } + return strings.Compare(a.recorded, b.recorded) + }) + return result + } + p.packagePaths = sorted(packages) + p.modulePaths = sorted(modules) +} + +// RecordedPath returns the source path to record in the output. +func (p *Package) RecordedPath(filename string) string { + if !p.program.config.TrimPath() { + return filename + } + if filepath.Base(filename) == "!cgo.go" { + return path.Join(p.RecordedDir(), "!cgo.go") + } + if p.program.config.TestConfig.CompileTestBinary { + for _, generated := range p.program.MainPkg().GoFiles { + if filename == generated { + return "_testmain.go" + } + } + } + for _, root := range []string{ + filepath.Join(p.program.goroot, "src"), + filepath.Join(goenv.Get("GOROOT"), "src"), + filepath.Join(goenv.Get("TINYGOROOT"), "src"), + } { + rel, err := filepath.Rel(root, filename) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return filepath.ToSlash(rel) + } + } + + var recordedPath string + var matchedDirLength int + for _, mapping := range p.program.packagePaths { + dir := mapping.original + if len(dir) < matchedDirLength { + break + } + rel, err := filepath.Rel(dir, filename) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + continue + } + candidate := path.Join(mapping.recorded, filepath.ToSlash(rel)) + if len(dir) > matchedDirLength || len(dir) == matchedDirLength && candidate < recordedPath { + matchedDirLength = len(dir) + recordedPath = candidate + } + } + if recordedPath != "" { + return recordedPath + } + for _, mapping := range p.program.modulePaths { + dir := mapping.original + if len(dir) < matchedDirLength { + break + } + rel, err := filepath.Rel(dir, filename) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + continue + } + candidate := path.Join(mapping.recorded, filepath.ToSlash(rel)) + if len(dir) > matchedDirLength || len(dir) == matchedDirLength && candidate < recordedPath { + matchedDirLength = len(dir) + recordedPath = candidate + } + } + if recordedPath != "" { + return recordedPath + } + if filepath.IsAbs(filename) { + return filename + } + return filepath.ToSlash(filename) +} + +// DebugPrefixMap returns the Clang flag that maps this package to its recorded +// source directory. +func (p *Package) DebugPrefixMap() string { + return "-ffile-prefix-map=" + p.OriginalModuleDir() + "=" + p.program.config.CSourcePath(p.RecordedModuleDir()) +} + +// RecordedCFlags returns C flags with path options rewritten to use recorded +// paths. +func (p *Package) RecordedCFlags() []string { + if !p.program.config.TrimPath() { + return p.CFlags + } + return compileopts.RewriteCFlags(p.CFlags, p.recordedCFlagPath) +} + +func (p *Package) recordedCFlagPath(filename string) string { + if recorded := p.RecordedPath(filename); recorded != filename { + return recorded + } + for _, root := range []struct{ from, to string }{ + {p.OriginalModuleDir(), p.RecordedModuleDir()}, + {goenv.Get("TINYGOROOT"), "github.com/tinygo-org/tinygo"}, + {goenv.Get("GOCACHE"), "tinygo-cache"}, + } { + rel, err := filepath.Rel(root.from, filename) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return path.Join(root.to, filepath.ToSlash(rel)) + } + } + return filename +} + // parseFile is a wrapper around parser.ParseFile. func (p *Package) parseFile(path string, mode parser.Mode) (*ast.File, error) { originalPath := p.program.getOriginalPath(path) @@ -513,7 +718,16 @@ func (p *Package) parseFiles() ([]*ast.File, error) { return nil, fmt.Errorf("failed to split CGO_CFLAGS: %w", err) } initialCFlags = append(initialCFlags, cgoCFlags...) - generated, headerCode, cflags, ldflags, accessedFiles, errs := cgo.Process(files, p.program.workingDir, p.ImportPath, p.program.fset, initialCFlags, p.program.config.GOOS()) + if p.program.config.TrimPath() { + initialCFlags = append(initialCFlags, + p.DebugPrefixMap(), + "-ffile-prefix-map="+filepath.Join(p.program.goroot, "src")+"="+p.program.config.CSourcePath("."), + ) + } + // LLVM 15's driver changes the process cwd. Set only FileManager's cwd. + // See clang/lib/Driver/Driver.cpp and clang/lib/Basic/FileManager.cpp. + parseFlags := append(append([]string(nil), initialCFlags...), "-Xclang", "-working-directory", "-Xclang", p.Dir) + generated, headerCode, cflags, ldflags, accessedFiles, errs := cgo.Process(files, p.program.workingDir, p.ImportPath, p.program.fset, parseFlags, p.program.config.GOOS()) p.CFlags = append(initialCFlags, cflags...) p.CGoHeaders = headerCode for path, hash := range accessedFiles { diff --git a/loader/loader_test.go b/loader/loader_test.go new file mode 100644 index 0000000000..1ea72bb64a --- /dev/null +++ b/loader/loader_test.go @@ -0,0 +1,222 @@ +package loader + +import ( + "fmt" + "path/filepath" + "slices" + "testing" + + "github.com/tinygo-org/tinygo/compileopts" + "github.com/tinygo-org/tinygo/goenv" +) + +func BenchmarkRecordedPath(b *testing.B) { + for _, count := range []int{10, 100, 1000} { + b.Run(fmt.Sprintf("packages=%d", count), func(b *testing.B) { + program := &Program{ + config: &compileopts.Config{ + Options: &compileopts.Options{TrimPath: true}, + }, + goroot: filepath.FromSlash("/goroot"), + Packages: make(map[string]*Package), + } + var pkg *Package + for i := 0; i < count; i++ { + pkg = &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash(fmt.Sprintf("/tmp/module/pkg%d", i)), + ImportPath: fmt.Sprintf("example.com/module/pkg%d", i), + }, + } + pkg.Module.Path = "example.com/module" + pkg.Module.Dir = filepath.FromSlash("/tmp/module") + program.Packages[pkg.ImportPath] = pkg + } + program.initRecordedPaths() + for _, name := range []string{"package", "module"} { + b.Run(name, func(b *testing.B) { + filename := filepath.Join(pkg.Dir, "file.go") + want := pkg.ImportPath + "/file.go" + if name == "module" { + filename = filepath.FromSlash("/tmp/module/include/shared.h") + want = "example.com/module/include/shared.h" + } + b.ReportAllocs() + for b.Loop() { + if got := pkg.RecordedPath(filename); got != want { + b.Fatalf("RecordedPath() = %q, want %q", got, want) + } + } + }) + } + }) + } +} + +func TestRecordedPathPrecedence(t *testing.T) { + program := &Program{ + config: &compileopts.Config{ + Options: &compileopts.Options{TrimPath: true}, + }, + goroot: filepath.FromSlash("/goroot"), + Packages: make(map[string]*Package), + } + var pkg *Package + for _, entry := range []struct{ dir, name, moduleDir, modulePath string }{ + {"/tmp/root/pkg", "example.com/z", "/tmp/root", "example.com/root"}, + {"/tmp/root/pkg", "example.com/a", "/tmp/root", "example.com/root"}, + {"/tmp/root/pkg/nested", "example.com/nested", "/tmp/root", "example.com/root"}, + {"/tmp/root/pkg/include/sub", "example.com/includes/sub", "/tmp/root/pkg/include", "example.com/includes"}, + {"/tmp/root/extra/sub", "example.com/extra/sub", "/tmp/root/extra", "example.com/extra"}, + } { + pkg = &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash(entry.dir), + ImportPath: entry.name, + }, + } + pkg.Module.Dir = filepath.FromSlash(entry.moduleDir) + pkg.Module.Path = entry.modulePath + program.Packages[pkg.ImportPath] = pkg + } + program.initRecordedPaths() + for _, tc := range []struct{ filename, want string }{ + {"/tmp/root/pkg/file.go", "example.com/a/file.go"}, + {"/tmp/root/pkg/nested/file.go", "example.com/nested/file.go"}, + {"/tmp/root/pkg/include/shared.h", "example.com/a/include/shared.h"}, + {"/tmp/root/extra/include/shared.h", "example.com/extra/include/shared.h"}, + {"/tmp/root/include/shared.h", "example.com/root/include/shared.h"}, + {"/tmp/root/pkg-other/file.go", "example.com/root/pkg-other/file.go"}, + {"/tmp/root-other/file.go", filepath.FromSlash("/tmp/root-other/file.go")}, + } { + if got := pkg.RecordedPath(filepath.FromSlash(tc.filename)); got != tc.want { + t.Errorf("RecordedPath(%q) = %q, want %q", tc.filename, got, tc.want) + } + } +} + +func TestRecordedPath(t *testing.T) { + config := &compileopts.Config{ + Options: &compileopts.Options{TrimPath: true}, + Target: &compileopts.TargetSpec{GOOS: "linux"}, + } + program := &Program{ + config: config, + goroot: filepath.FromSlash("/goroot"), + Packages: make(map[string]*Package), + } + pkg := &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash("/tmp/module"), + ImportPath: "example.com/main", + }, + } + pkg.Module.Path = "example.com/main" + pkg.Module.Dir = filepath.FromSlash("/tmp/module") + dependency := &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash("/tmp/dependency/subpackage"), + ImportPath: "example.com/dependency/subpackage", + }, + } + dependency.Module.Path = "example.com/dependency" + dependency.Module.Version = "v1.2.3" + dependency.Module.Dir = filepath.FromSlash("/tmp/dependency") + dependency.CFlags = []string{ + "-I" + filepath.FromSlash("/tmp/dependency/include"), + "-isystem", + filepath.FromSlash("/tmp/dependency/system"), + "-iquote" + filepath.FromSlash("/tmp/dependency/include"), + "-ffile-prefix-map=" + filepath.FromSlash("/tmp/dependency") + "=/mapped", + `-DCONFIG_PATH="/tmp/dependency"`, + } + program.Packages[pkg.ImportPath] = pkg + program.Packages[dependency.ImportPath] = dependency + program.initRecordedPaths() + + if got, want := dependency.RecordedDir(), "example.com/dependency@v1.2.3/subpackage"; got != want { + t.Fatalf("RecordedDir() = %q, want %q", got, want) + } + filename := filepath.FromSlash("/tmp/dependency/subpackage/file.go") + if got, want := pkg.RecordedPath(filename), "example.com/dependency@v1.2.3/subpackage/file.go"; got != want { + t.Fatalf("RecordedPath() = %q, want %q", got, want) + } + header := filepath.FromSlash("/tmp/dependency/include/shared.h") + if got, want := pkg.RecordedPath(header), "example.com/dependency@v1.2.3/include/shared.h"; got != want { + t.Fatalf("RecordedPath() = %q, want %q", got, want) + } + if got, want := dependency.DebugPrefixMap(), "-ffile-prefix-map="+filepath.FromSlash("/tmp/dependency")+"=/_/example.com/dependency@v1.2.3"; got != want { + t.Fatalf("DebugPrefixMap() = %q, want %q", got, want) + } + if got, want := dependency.RecordedCFlags(), []string{ + "-Iexample.com/dependency@v1.2.3/include", + "-isystem", + "example.com/dependency@v1.2.3/system", + "-iquoteexample.com/dependency@v1.2.3/include", + "-ffile-prefix-map=example.com/dependency@v1.2.3=/mapped", + `-DCONFIG_PATH="/tmp/dependency"`, + }; !slices.Equal(got, want) { + t.Fatalf("RecordedCFlags() = %q, want %q", got, want) + } + + for _, root := range []string{program.goroot, goenv.Get("GOROOT"), goenv.Get("TINYGOROOT")} { + filename := filepath.Join(root, "src", "runtime", "header.h") + if got := pkg.RecordedPath(filename); got != "runtime/header.h" { + t.Errorf("RecordedPath(%q) = %q", filename, got) + } + if got := pkg.recordedCFlagPath(filename); got != "runtime/header.h" { + t.Errorf("recordedCFlagPath(%q) = %q", filename, got) + } + } + outside := filepath.FromSlash("/tmp/module-other/include") + if got := pkg.recordedCFlagPath(outside); got != outside { + t.Errorf("rewrote a path outside the module: %q", got) + } + + vendored := &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash("/tmp/main/vendor/example.com/dependency/subpackage"), + ImportPath: "example.com/dependency/subpackage", + }, + } + vendored.Module.Path = "example.com/dependency" + vendored.Module.Version = "v1.2.3" + program.Packages[vendored.ImportPath] = vendored + program.initRecordedPaths() + if got, want := vendored.OriginalModuleDir(), filepath.FromSlash("/tmp/main/vendor/example.com/dependency"); got != want { + t.Fatalf("vendored OriginalModuleDir() = %q, want %q", got, want) + } + vendoredHeader := filepath.FromSlash("/tmp/main/vendor/example.com/dependency/include/shared.h") + if got, want := pkg.RecordedPath(vendoredHeader), "example.com/dependency@v1.2.3/include/shared.h"; got != want { + t.Fatalf("vendored RecordedPath() = %q, want %q", got, want) + } + if got, want := vendored.DebugPrefixMap(), "-ffile-prefix-map="+filepath.FromSlash("/tmp/main/vendor/example.com/dependency")+"=/_/example.com/dependency@v1.2.3"; got != want { + t.Fatalf("vendored DebugPrefixMap() = %q, want %q", got, want) + } + + gopath := &Package{ + program: program, + PackageJSON: PackageJSON{ + Dir: filepath.FromSlash("/gopath/src/example.com/dependency/subpackage"), + ImportPath: "example.com/dependency/subpackage", + Root: filepath.FromSlash("/gopath"), + }, + } + program.Packages[gopath.ImportPath] = gopath + program.initRecordedPaths() + if got, want := gopath.OriginalModuleDir(), filepath.FromSlash("/gopath/src"); got != want { + t.Fatalf("GOPATH OriginalModuleDir() = %q, want %q", got, want) + } + gopathHeader := filepath.FromSlash("/gopath/src/example.com/dependency/include/shared.h") + if got, want := pkg.RecordedPath(gopathHeader), "example.com/dependency/include/shared.h"; got != want { + t.Fatalf("GOPATH RecordedPath() = %q, want %q", got, want) + } + if got, want := gopath.DebugPrefixMap(), "-ffile-prefix-map="+filepath.FromSlash("/gopath/src")+"=/_"; got != want { + t.Fatalf("GOPATH DebugPrefixMap() = %q, want %q", got, want) + } +} diff --git a/main.go b/main.go index 321e0c3979..7e1dfee78e 100644 --- a/main.go +++ b/main.go @@ -1785,6 +1785,7 @@ func main() { target := flag.String("target", "", "chip/board name or JSON target specification file") buildMode := flag.String("buildmode", "", "build mode to use (default, c-shared, wasi-legacy)") buildVCS := flag.String("buildvcs", "auto", "whether to stamp version control information (true, false, auto)") + trimPath := flag.Bool("trimpath", false, "remove all file system paths from the resulting executable") var stackSize uint64 flag.Func("stack-size", "goroutine stack size (if unknown at compile time)", func(s string) error { size, err := bytesize.Parse(s) @@ -1920,6 +1921,7 @@ func main() { Target: *target, BuildMode: *buildMode, BuildVCS: *buildVCS, + TrimPath: *trimPath, StackSize: stackSize, Opt: *opt, GC: *gc, diff --git a/main_test.go b/main_test.go index d752baa0df..2a915347c0 100644 --- a/main_test.go +++ b/main_test.go @@ -9,14 +9,18 @@ import ( "context" "debug/dwarf" "debug/elf" + "debug/pe" "errors" "flag" + "fmt" "io" "os" "os/exec" + "path/filepath" "reflect" "regexp" "runtime" + "slices" "strings" "sync" "testing" @@ -50,6 +54,265 @@ var supportedLinuxArches = map[string]string{ var sema = make(chan struct{}, runtime.NumCPU()) +func TestTrimPath(t *testing.T) { + t.Setenv("CGO_CFLAGS", "-iquoteinclude -includestdint.h -imacros relative.h") + root := t.TempDir() + var binaries [][]byte + options := optionsFromTarget(*testTarget, sema) + config, err := builder.NewConfig(&options) + if err != nil { + t.Fatal(err) + } + + for _, name := range []string{"a", "b"} { + dir := filepath.Join(root, name) + if err := os.CopyFS(dir, os.DirFS(filepath.Join(TESTDATA, "trimpath"))); err != nil { + t.Fatal(err) + } + + outpath := filepath.Join(root, name+".out") + cache := filepath.Join(root, "cache-"+name) + cmd := trimPathCommand(t, dir, "build", "-trimpath", "-p=4", "-o", outpath, ".") + cmd.Env = append(cmd.Env, "XDG_CACHE_HOME="+cache, "HOME="+cache, "LocalAppData="+cache) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build failed: %v\n%s", err, output) + } + binary, err := os.ReadFile(outpath) + if err != nil { + t.Fatal(err) + } + for _, localPath := range []string{root, goenv.Get("GOROOT"), goenv.Get("TINYGOROOT"), goenv.Get("GOCACHE")} { + if bytes.Contains(binary, []byte(localPath)) || bytes.Contains(binary, []byte(filepath.ToSlash(localPath))) { + t.Errorf("trimmed binary contains local path %q", localPath) + } + } + binaries = append(binaries, binary) + } + + if !bytes.Equal(binaries[0], binaries[1]) { + t.Error("trimmed binaries built in different directories are not identical") + } + if config.GOOS() != "darwin" { + files := trimPathDWARFFiles(t, filepath.Join(root, "a.out"), config.GOOS()) + headerPath := "/_/example.com/dependency@v1.2.3/include/shared.h" + if config.GOOS() == "windows" { + headerPath = "//_/_/example.com/dependency@v1.2.3/include/shared.h" + } + for _, want := range []string{ + "example.com/trimpath/main.go", + "example.com/dependency@v1.2.3/subpackage/dependency.go", + headerPath, + } { + if !slices.Contains(files, want) { + t.Errorf("missing DWARF path %q in %v", want, files) + } + } + for _, file := range files { + if strings.Contains(file, filepath.ToSlash(root)) || strings.Count(file, "example.com/") > 1 { + t.Errorf("invalid DWARF path %q", file) + } + } + } + + dir := filepath.Join(root, "a") + for _, trim := range []bool{false, true} { + outpath := filepath.Join(root, "nodebug.out") + args := []string{"build", "-no-debug", "-o", outpath} + if trim { + args = append(args, "-trimpath") + } + cmd := trimPathCommand(t, dir, append(args, ".")...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build failed: %v\n%s", err, output) + } + binary, err := os.ReadFile(outpath) + if err != nil { + t.Fatal(err) + } + containsPath := bytes.Contains(binary, []byte(dir)) || bytes.Contains(binary, []byte(filepath.ToSlash(dir))) + if containsPath == trim { + t.Errorf("trimpath=%v: contains local __FILE__ path=%v", trim, containsPath) + } + } +} + +func TestTrimPathStackSizes(t *testing.T) { + dir := t.TempDir() + source := `package main +func main() { + done := make(chan int) + go func() { done <- 42 }() + println(<-done) +} +` + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte(source), 0o600); err != nil { + t.Fatal(err) + } + for _, trim := range []bool{false, true} { + t.Run(fmt.Sprintf("trimpath=%v", trim), func(t *testing.T) { + outpath := filepath.Join(dir, "main.elf") + args := []string{"build", "-target=pico2", "-o", outpath} + if trim { + args = append(args, "-trimpath") + } + cmd := trimPathCommand(t, dir, append(args, "main.go")...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build failed: %v\n%s", err, output) + } + file, err := elf.Open(outpath) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if section := file.Section(".tinygo_stacksizes"); section == nil || section.Size == 0 { + t.Fatal("missing automatic stack-size data") + } + binary, err := os.ReadFile(outpath) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(binary, []byte("task_stack.go")) { + t.Fatal("missing stack-size source file") + } + sourceDir := filepath.Join(goenv.Get("TINYGOROOT"), "src") + containsPath := bytes.Contains(binary, []byte(sourceDir)) || bytes.Contains(binary, []byte(filepath.ToSlash(sourceDir))) + if containsPath == trim { + t.Errorf("trimpath=%v: contains local runtime source directory=%v", trim, containsPath) + } + }) + } +} + +func trimPathCommand(t *testing.T, dir string, args ...string) *exec.Cmd { + t.Helper() + options := optionsFromTarget(*testTarget, sema) + if options.Target != "" { + args = slices.Insert(args, 1, "-target="+options.Target) + } + cmd := exec.Command(os.Args[0], append([]string{"test-main"}, args...)...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "TINYGOROOT="+goenv.Get("TINYGOROOT"), + "GOPATH="+goenv.Get("GOPATH"), + "GOOS="+options.GOOS, + "GOARCH="+options.GOARCH, + "GOARM="+options.GOARM, + ) + return cmd +} + +func trimPathDWARFFiles(t *testing.T, filename, goos string) []string { + t.Helper() + f, err := os.Open(filename) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var data *dwarf.Data + switch goos { + case "windows": + file, parseErr := pe.NewFile(f) + if parseErr != nil { + t.Fatal(parseErr) + } + data, err = file.DWARF() + case "wasip1", "wasip2", "js": + file, parseErr := wasm.Parse(f) + if parseErr != nil { + t.Fatal(parseErr) + } + data, err = file.DWARF() + default: + file, parseErr := elf.NewFile(f) + if parseErr != nil { + t.Fatal(parseErr) + } + data, err = file.DWARF() + } + if err != nil { + t.Fatal(err) + } + var files []string + reader := data.Reader() + for { + entry, err := reader.Next() + if err != nil { + t.Fatal(err) + } + if entry == nil { + break + } + if entry.Tag != dwarf.TagCompileUnit { + continue + } + lines, err := data.LineReader(entry) + if err != nil { + t.Fatal(err) + } + if lines == nil { + continue + } + var line dwarf.LineEntry + for { + if err := lines.Next(&line); err != nil { + if err != io.EOF { + t.Fatal(err) + } + break + } + } + for _, file := range lines.Files() { + if file != nil { + files = append(files, strings.ReplaceAll(file.Name, "\\", "/")) + } + } + } + return files +} + +func TestTrimPathTestPackages(t *testing.T) { + options := optionsFromTarget(*testTarget, sema) + config, err := builder.NewConfig(&options) + if err != nil { + t.Fatal(err) + } + if config.GOOS() == "darwin" { + t.Skip("Mach-O executables keep DWARF in separate object files") + } + for _, externalOnly := range []bool{false, true} { + dir := t.TempDir() + files := map[string]string{ + "go.mod": "module example.com/pkg_test\n\ngo 1.23\n", + "external_test.go": "package pkg_test\nimport \"testing\"\nfunc TestPath(t *testing.T) { t.Log(\"path\") }\n", + } + if !externalOnly { + files["pkg.go"] = "package pkg\nvar Value = 1\n" + } + for name, data := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o644); err != nil { + t.Fatal(err) + } + } + outpath := filepath.Join(dir, "test.out") + cmd := trimPathCommand(t, dir, "test", "-trimpath", "-c", "-o", outpath, ".") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("test build failed: %v\n%s", err, output) + } + got := trimPathDWARFFiles(t, outpath, config.GOOS()) + if !slices.Contains(got, "example.com/pkg_test/external_test.go") { + t.Errorf("externalOnly=%v: incorrect test paths: %v", externalOnly, got) + } + if !slices.Contains(got, "_testmain.go") { + t.Errorf("externalOnly=%v: missing generated test main path", externalOnly) + } + for _, file := range got { + if filepath.IsAbs(filepath.FromSlash(file)) && !strings.HasPrefix(file, "/_/") && !strings.HasPrefix(file, "//_/_/") { + t.Errorf("externalOnly=%v: untrimmed test path %q", externalOnly, file) + } + } + } +} + func TestBuild(t *testing.T) { t.Parallel() @@ -1573,6 +1836,11 @@ func TestMain(m *testing.M) { os.Exit(1) } os.Exit(0) + case "test-main": + os.Args = append([]string{os.Args[0]}, os.Args[2:]...) + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) + main() + os.Exit(0) } } diff --git a/make/config.mk b/make/config.mk index 5b592d8261..020024e4ae 100644 --- a/make/config.mk +++ b/make/config.mk @@ -36,7 +36,7 @@ GO ?= go # Flags to pass to go test. GOTESTFLAGS ?= -GOTESTPKGS ?= ./builder ./cgo ./compileopts ./compiler ./interp ./transform . +GOTESTPKGS ?= ./builder ./cgo ./compileopts ./compiler ./interp ./loader ./transform . # tinygo binary for tests TINYGO ?= $(call detect,tinygo,tinygo $(CURDIR)/build/tinygo) diff --git a/testdata/trimpath/data.txt b/testdata/trimpath/data.txt new file mode 100644 index 0000000000..80e21b7cb6 --- /dev/null +++ b/testdata/trimpath/data.txt @@ -0,0 +1 @@ +embedded diff --git a/testdata/trimpath/dependency/go.mod b/testdata/trimpath/dependency/go.mod new file mode 100644 index 0000000000..5150d6776e --- /dev/null +++ b/testdata/trimpath/dependency/go.mod @@ -0,0 +1,3 @@ +module example.com/dependency + +go 1.23 diff --git a/testdata/trimpath/dependency/include/shared.h b/testdata/trimpath/dependency/include/shared.h new file mode 100644 index 0000000000..c35cf87dd7 --- /dev/null +++ b/testdata/trimpath/dependency/include/shared.h @@ -0,0 +1,7 @@ +static inline int sharedValueInline(void) { + return 7; +} + +static inline const char *sharedHeaderPathInline(void) { + return __FILE__; +} diff --git a/testdata/trimpath/dependency/subpackage/dependency.c b/testdata/trimpath/dependency/subpackage/dependency.c new file mode 100644 index 0000000000..f6fc682324 --- /dev/null +++ b/testdata/trimpath/dependency/subpackage/dependency.c @@ -0,0 +1,12 @@ +#include "shared.h" +#include "relative.h" + +_Static_assert(RELATIVE_VALUE == 43, "incorrect relative include"); + +int sharedValue(void) { + return sharedValueInline(); +} + +const char *sharedHeaderPath(void) { + return sharedHeaderPathInline(); +} diff --git a/testdata/trimpath/dependency/subpackage/dependency.go b/testdata/trimpath/dependency/subpackage/dependency.go new file mode 100644 index 0000000000..3dac0ee8cc --- /dev/null +++ b/testdata/trimpath/dependency/subpackage/dependency.go @@ -0,0 +1,16 @@ +package dependency + +/* +#cgo CFLAGS: -I../include -Iinclude +#include "relative.h" +_Static_assert(RELATIVE_VALUE == 43, "incorrect relative include"); +int sharedValue(void); +const char *sharedHeaderPath(void); +*/ +import "C" + +//go:noinline +func Value() int { return int(C.sharedValue()) } + +//go:noinline +func HeaderPath() string { return C.GoString(C.sharedHeaderPath()) } diff --git a/testdata/trimpath/dependency/subpackage/include/relative.h b/testdata/trimpath/dependency/subpackage/include/relative.h new file mode 100644 index 0000000000..63cc107be8 --- /dev/null +++ b/testdata/trimpath/dependency/subpackage/include/relative.h @@ -0,0 +1 @@ +#define RELATIVE_VALUE 43 diff --git a/testdata/trimpath/go.mod b/testdata/trimpath/go.mod new file mode 100644 index 0000000000..1c1d89052a --- /dev/null +++ b/testdata/trimpath/go.mod @@ -0,0 +1,7 @@ +module example.com/trimpath + +go 1.23 + +require example.com/dependency v1.2.3 + +replace example.com/dependency => ./dependency diff --git a/testdata/trimpath/include/relative.h b/testdata/trimpath/include/relative.h new file mode 100644 index 0000000000..6c49439eea --- /dev/null +++ b/testdata/trimpath/include/relative.h @@ -0,0 +1 @@ +#define RELATIVE_VALUE 42 diff --git a/testdata/trimpath/main.c b/testdata/trimpath/main.c new file mode 100644 index 0000000000..28b992f873 --- /dev/null +++ b/testdata/trimpath/main.c @@ -0,0 +1,5 @@ +#include "relative.h" +#include + +_Static_assert(RELATIVE_VALUE == 42, "incorrect relative include"); +int32_t value(void) { return RELATIVE_VALUE; } diff --git a/testdata/trimpath/main.go b/testdata/trimpath/main.go new file mode 100644 index 0000000000..4b55529869 --- /dev/null +++ b/testdata/trimpath/main.go @@ -0,0 +1,22 @@ +package main + +/* +#cgo CFLAGS: -Iinclude +int value(void); +static int inlineValue(void) { return 1; } +*/ +import "C" + +import ( + _ "embed" + "runtime" + + "example.com/dependency/subpackage" +) + +//go:embed data.txt +var data string + +func main() { + println(C.value(), C.inlineValue(), dependency.Value(), dependency.HeaderPath(), data, runtime.GOROOT()) +} diff --git a/transform/stacksize.go b/transform/stacksize.go index 2f7a6c1d6f..8d1d72bc6a 100644 --- a/transform/stacksize.go +++ b/transform/stacksize.go @@ -81,9 +81,13 @@ func CreateStackSizeLoads(mod llvm.Module, config *compileopts.Config) []string }, }, }) + sourceDir := "" + if !config.TrimPath() { + sourceDir = filepath.Join(goenv.Get("TINYGOROOT"), "src") + } diglobal := dibuilder.CreateGlobalVariableExpression(llvm.Metadata{}, llvm.DIGlobalVariableExpression{ Name: "internal/task.stackSizes", - File: dibuilder.CreateFile("internal/task/task_stack.go", filepath.Join(goenv.Get("TINYGOROOT"), "src")), + File: dibuilder.CreateFile("internal/task/task_stack.go", sourceDir), Line: 1, Type: ditype, Expr: dibuilder.CreateExpression(nil),