diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5db17ebcc..3c9ac6662 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - go_version: [1.19.x, 1.20.x] + go_version: [1.24.x, 1.25.x, 1.26.x, 1.27.x] runs-on: ${{ matrix.os }} steps: - name: Install Go ${{ matrix.go_version }} diff --git a/.github/workflows/verify-go-src.yml b/.github/workflows/verify-go-src.yml index 4f80a3c22..0b4f3867b 100644 --- a/.github/workflows/verify-go-src.yml +++ b/.github/workflows/verify-go-src.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - go_version: [1.20.x] + go_version: [1.27.x] runs-on: ${{ matrix.os }} steps: - name: Install Go ${{ matrix.go_version }} diff --git a/go.mod b/go.mod index e5c49bfe9..d7f8b5d3a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/netlify/open-api/v2 -go 1.13 +go 1.24 require ( github.com/Azure/go-autorest/autorest v0.10.1 @@ -15,5 +15,52 @@ require ( github.com/rsc/goversion v1.2.0 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.8.2 + golang.org/x/sys v0.5.0 +) + +require ( + github.com/Azure/go-autorest/autorest/adal v0.8.2 // indirect + github.com/Azure/go-autorest/autorest/date v0.2.0 // indirect + github.com/Azure/go-autorest/logger v0.1.0 // indirect + github.com/Azure/go-autorest/tracing v0.5.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/go-openapi/analysis v0.19.16 // indirect + github.com/go-openapi/inflect v0.19.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/loads v0.20.0 // indirect + github.com/go-openapi/spec v0.20.0 // indirect + github.com/go-stack/stack v1.8.0 // indirect + github.com/gorilla/handlers v1.4.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jessevdk/go-flags v1.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/kr/pretty v0.2.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/magiconair/properties v1.8.1 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/mitchellh/mapstructure v1.4.0 // indirect + github.com/pelletier/go-toml v1.8.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/afero v1.2.2 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.7.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/toqueteos/webbrowser v1.2.0 // indirect + go.mongodb.org/mongo-driver v1.4.4 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.7.0 // indirect + golang.org/x/text v0.7.0 // indirect + golang.org/x/tools v0.1.12 // indirect + gopkg.in/ini.v1 v1.57.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ae500ec3c..ff6d4247a 100644 --- a/go.sum +++ b/go.sum @@ -420,7 +420,6 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -448,7 +447,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -494,8 +492,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -509,7 +505,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -533,21 +528,13 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200610111108-226ff32320da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -584,7 +571,6 @@ golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= diff --git a/go/porcelain/deploy.go b/go/porcelain/deploy.go index 038e488b4..9fa393876 100644 --- a/go/porcelain/deploy.go +++ b/go/porcelain/deploy.go @@ -139,6 +139,11 @@ type FileBundle struct { // pathReader is lazily opened from Path when Buffer is nil, so the deprecated Read/Seek/Close // methods keep working for external callers that treat a FileBundle as an io.ReadSeekCloser. pathReader *os.File + + // root is the path that Path should be resolved relative to. When we read the contents of the + // file at Path, we read it using [os.OpenInRoot] to prevent symlinks from resolving to a file + // outside of the root: This is that root. + root string } type FunctionMetadata struct { @@ -602,14 +607,18 @@ func (n *Netlify) uploadFile(ctx context.Context, d *models.Deploy, f *FileBundl } sharedErr.mutex.Unlock() - var operationError error + relPath, relErr := filepath.Rel(f.root, f.Path) + if relErr != nil { + return backoff.Permanent(relErr) + } + var operationError error switch t { case fileUpload: var body io.ReadCloser - body, operationError = os.Open(f.Path) + body, operationError = os.OpenInRoot(f.root, relPath) if operationError == nil { - defer body.Close() + defer func() { _ = body.Close() }() params := operations.NewUploadDeployFileParams().WithDeployID(d.ID).WithPath(f.Name).WithFileBody(body) if f.Size != nil { params.WithSize(f.Size) @@ -621,9 +630,9 @@ func (n *Netlify) uploadFile(ctx context.Context, d *models.Deploy, f *FileBundl } case functionUpload: var body io.ReadCloser - body, operationError = os.Open(f.Path) + body, operationError = os.OpenInRoot(f.root, relPath) if operationError == nil { - defer body.Close() + defer func() { _ = body.Close() }() params := operations.NewUploadDeployFunctionParams().WithDeployID(d.ID).WithName(f.Name).WithFileBody(body).WithRuntime(&f.Runtime) if retryCount > 0 { @@ -642,9 +651,9 @@ func (n *Netlify) uploadFile(ctx context.Context, d *models.Deploy, f *FileBundl } case edgeFunctionUpload: var body io.ReadCloser - body, operationError = os.Open(f.Path) + body, operationError = os.OpenInRoot(f.root, relPath) if operationError == nil { - defer body.Close() + defer func() { _ = body.Close() }() params := operations.NewUploadDeployEdgeFunctionParams().WithDeployID(d.ID).WithCodeSha(f.Sum).WithFileBody(body) if retryCount > 0 { params = params.WithXNfRetryCount(&retryCount) @@ -697,24 +706,30 @@ func (n *Netlify) uploadFile(ctx context.Context, d *models.Deploy, f *FileBundl } } -func createFileBundle(rel, path string) (*FileBundle, error) { - return createFileBundleWithHasher(rel, path, sha1.New()) +func createFileBundle(rel, path, root string) (*FileBundle, error) { + return createFileBundleWithHasher(rel, path, root, sha1.New()) } -func createFunctionFileBundle(rel, path string) (*FileBundle, error) { - return createFileBundleWithHasher(rel, path, sha256.New()) +func createFunctionFileBundle(rel, path, root string) (*FileBundle, error) { + return createFileBundleWithHasher(rel, path, root, sha256.New()) } -func createFileBundleWithHasher(rel, path string, s hash.Hash) (*FileBundle, error) { - o, err := os.Open(path) +func createFileBundleWithHasher(rel, path, root string, s hash.Hash) (*FileBundle, error) { + relPath, err := filepath.Rel(root, path) + if err != nil { + return nil, err + } + + o, err := os.OpenInRoot(root, relPath) if err != nil { return nil, err } - defer o.Close() + defer func() { _ = o.Close() }() file := &FileBundle{ Name: rel, Path: path, + root: root, } if _, err := io.Copy(s, o); err != nil { @@ -745,13 +760,13 @@ func walk(dir string, observer DeployObserver, useLargeMedia, ignoreInstallDirs return nil } - file, err := createFileBundle(rel, path) + file, err := createFileBundle(rel, path, dir) if err != nil { return err } if useLargeMedia { - o, err := os.Open(path) + o, err := os.OpenInRoot(dir, osRel) if err != nil { return err } @@ -799,7 +814,7 @@ func addInternalFilesToDeploy(dir, internalPath string, files *deployFiles, obse } rel := internalPath + forceSlashSeparators(osRel) - file, err := createFileBundle(rel, path) + file, err := createFileBundle(rel, path, dir) if err != nil { return err } @@ -845,7 +860,7 @@ func bundle(ctx context.Context, functionDir string, tmpDir *lazyTempDir, observ return nil, nil, nil, nil } - manifestFile, err := os.Open(filepath.Join(functionDir, "manifest.json")) + manifestFile, err := os.OpenInRoot(functionDir, "manifest.json") // If a `manifest.json` file is found, we extract the functions and their // metadata from it. @@ -867,23 +882,23 @@ func bundle(ctx context.Context, functionDir string, tmpDir *lazyTempDir, observ switch { case zipFile(i): - runtime, err := readZipRuntime(filePath) + runtime, err := readZipRuntime(functionDir, filePath) if err != nil { return nil, nil, nil, err } - file, err := newFunctionFile(filePath, i, runtime, nil, tmpDir, observer) + file, err := newFunctionFile(functionDir, filePath, i, runtime, nil, tmpDir, observer) if err != nil { return nil, nil, nil, err } functions.Add(file.Name, file) case jsFile(i): - file, err := newFunctionFile(filePath, i, jsRuntime, nil, tmpDir, observer) + file, err := newFunctionFile(functionDir, filePath, i, jsRuntime, nil, tmpDir, observer) if err != nil { return nil, nil, nil, err } functions.Add(file.Name, file) case goFile(filePath, i, observer): - file, err := newFunctionFile(filePath, i, amazonLinux2, nil, tmpDir, observer) + file, err := newFunctionFile(functionDir, filePath, i, amazonLinux2, nil, tmpDir, observer) if err != nil { return nil, nil, nil, err } @@ -935,7 +950,7 @@ func bundleFromManifest(ctx context.Context, manifestFile *os.File, tmpDir *lazy InvocationMode: function.InvocationMode, Timeout: function.Timeout, } - file, err := newFunctionFile(function.Path, fileInfo, runtime, &meta, tmpDir, observer) + file, err := newFunctionFile(filepath.Dir(function.Path), function.Path, fileInfo, runtime, &meta, tmpDir, observer) if err != nil { return nil, nil, nil, err } @@ -1008,12 +1023,27 @@ func bundleFromManifest(ctx context.Context, manifestFile *os.File, tmpDir *lazy return functions, schedules, functionsConfig, nil } -func readZipRuntime(filePath string) (string, error) { - zf, err := zip.OpenReader(filePath) +func readZipRuntime(root, filePath string) (string, error) { + relPath, err := filepath.Rel(root, filePath) + if err != nil { + return "", err + } + + f, err := os.OpenInRoot(root, relPath) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + + info, err := f.Stat() + if err != nil { + return "", err + } + + zf, err := zip.NewReader(f, info.Size()) if err != nil { return "", err } - defer zf.Close() for _, file := range zf.File { if file.Name == "netlify-toolchain" { @@ -1023,7 +1053,7 @@ func readZipRuntime(filePath string) (string, error) { // This preserves the current behavior in this library. return jsRuntime, nil } - defer fc.Close() + defer func() { _ = fc.Close() }() var tc toolchainSpec if err := json.NewDecoder(fc).Decode(&tc); err != nil { @@ -1038,15 +1068,15 @@ func readZipRuntime(filePath string) (string, error) { return jsRuntime, nil } -func newFunctionFile(filePath string, i os.FileInfo, runtime string, metadata *FunctionMetadata, tmpDir *lazyTempDir, observer DeployObserver) (*FileBundle, error) { +func newFunctionFile(root, filePath string, i os.FileInfo, runtime string, metadata *FunctionMetadata, tmpDir *lazyTempDir, observer DeployObserver) (*FileBundle, error) { var file *FileBundle var err error if zipFile(i) || tarFile(i) { name := strings.TrimSuffix(i.Name(), filepath.Ext(i.Name())) - file, err = createFunctionFileBundle(name, filePath) + file, err = createFunctionFileBundle(name, filePath, root) } else { - file, err = zipFunctionFile(filePath, i, runtime, tmpDir) + file, err = zipFunctionFile(root, filePath, i, runtime, tmpDir) } if err != nil { return nil, err @@ -1064,12 +1094,17 @@ func newFunctionFile(filePath string, i os.FileInfo, runtime string, metadata *F return file, nil } -func zipFunctionFile(filePath string, i os.FileInfo, runtime string, tmpDir *lazyTempDir) (*FileBundle, error) { - src, err := os.Open(filePath) +func zipFunctionFile(root, filePath string, i os.FileInfo, runtime string, tmpDir *lazyTempDir) (*FileBundle, error) { + relPath, err := filepath.Rel(root, filePath) if err != nil { return nil, err } - defer src.Close() + + src, err := os.OpenInRoot(root, relPath) + if err != nil { + return nil, err + } + defer func() { _ = src.Close() }() dir, err := tmpDir.get() if err != nil { @@ -1110,6 +1145,7 @@ func zipFunctionFile(filePath string, i os.FileInfo, runtime string, tmpDir *laz Name: strings.TrimSuffix(i.Name(), filepath.Ext(i.Name())), Sum: hex.EncodeToString(s.Sum(nil)), Path: tmpName, + root: filepath.Dir(tmpName), }, nil } @@ -1122,13 +1158,18 @@ func bundleEdgeFunctions(ctx context.Context, edgeFunctionsDir string, observer return nil, nil } - manifestBytes, err := os.ReadFile(filepath.Join(edgeFunctionsDir, "manifest.json")) + manifestFile, err := os.OpenInRoot(edgeFunctionsDir, "manifest.json") if os.IsNotExist(err) { return nil, nil } if err != nil { return nil, err } + manifestBytes, err := io.ReadAll(manifestFile) + manifestFile.Close() + if err != nil { + return nil, err + } context.GetLogger(ctx).Debug("Found edge functions manifest file") @@ -1166,7 +1207,7 @@ func newEdgeFunctionFile(edgeFunctionsDir string, bundle edgeFunctionsManifestBu // bundle's bytes rather than trusting the edge-bundler's asset filename (which currently also happens // to be the sha256, but that's a bundler implementation detail). createFileBundleWithHasher streams // the bytes through the hasher, so the bundle is never held in memory. - file, err := createFileBundleWithHasher(bundle.Format, path, sha256.New()) + file, err := createFileBundleWithHasher(bundle.Format, path, edgeFunctionsDir, sha256.New()) if err != nil { return nil, fmt.Errorf("edge functions manifest specifies a bundle that cannot be read: %s: %w", bundle.Asset, err) } diff --git a/go/porcelain/deploy_dirswap_linux_test.go b/go/porcelain/deploy_dirswap_linux_test.go new file mode 100644 index 000000000..8d50d8b22 --- /dev/null +++ b/go/porcelain/deploy_dirswap_linux_test.go @@ -0,0 +1,149 @@ +//go:build linux + +package porcelain_test + +import ( + gocontext "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + apiClient "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/netlify/open-api/v2/go/porcelain" + "github.com/netlify/open-api/v2/go/porcelain/context" +) + +// TestDoDeploy_SymlinkedParentTOCTOURace covers the parent-directory variant of the file-swap race: a +// deploy path that stays inside the tree by name is redirected out of it by replacing a parent +// directory with a symlink to a location outside the tree, between the walk and the checksum read. +// Resolving each read within the deploy root (os.OpenInRoot) refuses it, because the swapped-in +// symlink target escapes that root. +// +// The swap uses RENAME_EXCHANGE, which is Linux-only, so the test is too; the defense itself is not. +// +// The test asserts on the checksum (walk->hash) window only, which is the window openat2 uniquely +// closes: if the swap wins, the file's declared checksum is that of the out-of-tree file. The mock +// requires no files, so DoDeploy never enters the upload phase. This test is Linux-only because both +// the swap primitive (RENAME_EXCHANGE) and the defense (openat2) are. +func TestDoDeploy_SymlinkedParentTOCTOURace(t *testing.T) { + const deployID = "deploy-1" + + root := t.TempDir() + + // Contents of a file outside the deploy tree that a swapped-in parent symlink points at. If the + // walk->hash race is won, this file's checksum is what gets declared for an in-tree path. + outsideContents := []byte("OUTSIDE-TREE-CONTENTS" + strings.Repeat("A", 4096)) + outsideDigest := sha1.Sum(outsideContents) + outsideSHA1 := hex.EncodeToString(outsideDigest[:]) + + var fileswapsMu sync.Mutex + fileswaps := 0 + recordFileswap := func() { + fileswapsMu.Lock() + defer fileswapsMu.Unlock() + fileswaps++ + } + + mux := http.NewServeMux() + mux.HandleFunc("POST /api/v1/sites/{site_id}/deploys", func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var data struct { + Files map[string]string `json:"files"` + } + _ = json.Unmarshal(body, &data) + for _, sum := range data.Files { + if strings.HasPrefix(sum, outsideSHA1) { + recordFileswap() + } + } + // Require nothing so DoDeploy returns before uploading + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": deployID, + "site_id": r.PathValue("site_id"), + "state": "ready", + }) + }) + server := httptest.NewServer(mux) + defer server.Close() + + hu, err := url.Parse(server.URL) + require.NoError(t, err) + tr := apiClient.NewWithClient(hu.Host, "/api/v1", []string{"http"}, http.DefaultClient) + client := porcelain.NewRetryable(tr, strfmt.Default, 1) + client.SetSyncFileLimit(1_000_000) + ctx := context.WithAuthInfo(gocontext.Background(), apiClient.BearerToken("token")) + + for round := range 6 { + roundRoot := filepath.Join(root, fmt.Sprintf("round-%d", round)) + require.NoError(t, os.MkdirAll(roundRoot, 0o755)) + + publicDir := filepath.Join(roundRoot, "public") + require.NoError(t, os.Mkdir(publicDir, 0o755)) + + decoy := filepath.Join(roundRoot, "decoy") + require.NoError(t, os.Mkdir(decoy, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(decoy, "f"), outsideContents, 0o644)) + + stop := make(chan struct{}) + var wg sync.WaitGroup + + for i := range 40 { + // A real directory holding a benign file the walk records, and an alternate symlink to the + // decoy. Both live in the same parent so their names can be exchanged atomically. + realDir := filepath.Join(publicDir, fmt.Sprintf("d%03d", i)) + require.NoError(t, os.Mkdir(realDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(realDir, "f"), fmt.Appendf([]byte{}, "benign-%d-%d", round, i), 0o644)) + altLink := filepath.Join(publicDir, fmt.Sprintf("d%03d-alt", i)) + require.NoError(t, os.Symlink(decoy, altLink)) + + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + // Atomically replace the directory with a symlink. (Rename makes it atomic; a nonatomic + // version would make the directory momentarily disappear, which could result in an + // unrelated ENOENT error instead.) + _ = unix.Renameat2(unix.AT_FDCWD, realDir, unix.AT_FDCWD, altLink, unix.RENAME_EXCHANGE) + } + }() + } + + opts := &porcelain.DeployOptions{ + SiteID: "site-1", + Dir: publicDir, + } + // We can ignore errors here--this test is only concerned with checking if DoDeploy read files + // outside of the build directory root. + // nolint:errcheck + client.DoDeploy(ctx, opts, nil) + + close(stop) + wg.Wait() + } + + fileswapsMu.Lock() + defer fileswapsMu.Unlock() + if fileswaps > 0 { + t.Fatalf("DoDeploy hashed out-of-tree contents through a symlinked parent directory %d time(s)", fileswaps) + } +} diff --git a/go/porcelain/deploy_linux_test.go b/go/porcelain/deploy_linux_test.go new file mode 100644 index 000000000..3a3620ffb --- /dev/null +++ b/go/porcelain/deploy_linux_test.go @@ -0,0 +1,394 @@ +//go:build linux + +package porcelain_test + +import ( + gocontext "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + apiClient "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" + "github.com/netlify/open-api/v2/go/porcelain" + "github.com/netlify/open-api/v2/go/porcelain/context" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestDoDeploy_SymlinkTOCTOURace(t *testing.T) { + const fileswapMarker = "SWAPPED-FILE-MARKER" + const deployID = "deploy-1" + + const ( + edgeFunctionsInternalPath = ".netlify/internal/edge-functions/" + edgeRedirectsInternalPath = ".netlify/deploy-config/" + dbMigrationsInternalPath = ".netlify/internal/db/migrations/" + ) + + root := t.TempDir() + + // Create a file outside of the deploy directory. We'll attempt to replace files in the deploy + // tree with a symlink to this file while the upload is running and verify that the uploader + // doesn't allow this. + fileswapContents := []byte(fileswapMarker + strings.Repeat("A", 4096)) + fileswapPath := filepath.Join(root, "secret") + require.NoError(t, os.WriteFile(fileswapPath, fileswapContents, 0o600)) + fileswapSHA1Digest := sha1.Sum(fileswapContents) + fileswapSHA256Digest := sha256.Sum256(fileswapContents) + fileswapSHA1 := hex.EncodeToString(fileswapSHA1Digest[:]) + fileswapSHA256 := hex.EncodeToString(fileswapSHA256Digest[:]) + + var fileswapsMu sync.Mutex + fileswaps := map[string]int{} + recordFileswap := func(branch string) { + fileswapsMu.Lock() + defer fileswapsMu.Unlock() + fileswaps[branch]++ + } + + getUploadOriginForPath := func(name string) string { + if dec, err := url.PathUnescape(name); err == nil { + name = dec + } + switch { + case strings.HasPrefix(name, edgeFunctionsInternalPath): + return "edge-functions" + case strings.HasPrefix(name, edgeRedirectsInternalPath): + return "edge redirects" + case strings.HasPrefix(name, dbMigrationsInternalPath): + return "db migrations" + default: + return "publish dir" + } + } + + uniqueValues := func(m map[string]string) []string { + seen := make(map[string]struct{}, len(m)) + out := make([]string, 0, len(m)) + for _, v := range m { + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + out = append(out, v) + } + return out + } + + writeJSON := func(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) + } + // uploadHit records an upload-window leak: the client streamed the swapped path at upload time and + // its bytes carry the secret marker. This is the actual exfiltration channel the ticket describes. + uploadHit := func(branch string, r *http.Request) { + body, _ := io.ReadAll(r.Body) + if strings.Contains(string(body), fileswapMarker) { + recordFileswap("UPLOAD: " + branch) + } + } + + // Mock CreateSiteDeploy and UpdateSiteDeploy for this particular deploy + mux := http.NewServeMux() + mux.HandleFunc("POST /api/v1/sites/{site_id}/deploys", func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var data struct { + Files map[string]string `json:"files"` + Functions map[string]string `json:"functions"` + EdgeFunctions map[string]string `json:"edge_functions"` + } + err := json.Unmarshal(body, &data) + require.NoError(t, err) + + for name, sum := range data.Files { + if strings.HasPrefix(sum, fileswapSHA1) { + recordFileswap("HASH: " + getUploadOriginForPath(name)) + } + } + for _, sum := range data.Functions { + if sum == fileswapSHA256 { + recordFileswap("HASH: functions (bundle)") + } + } + for _, sum := range data.EdgeFunctions { + if sum == fileswapSHA256 { + recordFileswap("HASH: edge function bundle (bundleEdgeFunctions)") + } + } + + // Require all files/functions/etc. to test all upload file paths for fileswapping + writeJSON(w, map[string]any{ + "id": deployID, + "site_id": r.PathValue("site_id"), + "state": "uploading", + "required": uniqueValues(data.Files), + "required_functions": uniqueValues(data.Functions), + "required_edge_functions": uniqueValues(data.EdgeFunctions), + }) + }) + mux.HandleFunc(fmt.Sprintf("PUT /api/v1/deploys/%s/files/{path...}", deployID), func(w http.ResponseWriter, r *http.Request) { + uploadHit(getUploadOriginForPath(r.PathValue("path"))+" -> fileUpload", r) + writeJSON(w, map[string]any{}) + }) + mux.HandleFunc(fmt.Sprintf("PUT /api/v1/deploys/%s/functions/{name}", deployID), func(w http.ResponseWriter, r *http.Request) { + uploadHit("functions (bundle -> functionUpload)", r) + writeJSON(w, map[string]any{}) + }) + mux.HandleFunc(fmt.Sprintf("PUT /api/v1/deploys/%s/edge_functions/{code_sha}", deployID), func(w http.ResponseWriter, r *http.Request) { + uploadHit("edge function bundle (bundleEdgeFunctions -> edgeFunctionUpload)", r) + writeJSON(w, map[string]any{}) + }) + server := httptest.NewServer(mux) + defer server.Close() + + hu, err := url.Parse(server.URL) + require.NoError(t, err) + tr := apiClient.NewWithClient(hu.Host, "/api/v1", []string{"http"}, http.DefaultClient) + client := porcelain.NewRetryable(tr, strfmt.Default, 1) + // Use an artificially high limit to prevent the deploy from going into async mode + client.SetSyncFileLimit(1_000_000) + ctx := context.WithAuthInfo(gocontext.Background(), apiClient.BearerToken("token")) + + for round := range 4 { + roundRoot := filepath.Join(root, fmt.Sprintf("round-%d", round)) + require.NoError(t, os.MkdirAll(roundRoot, 0o755)) + staging := filepath.Join(roundRoot, "staging") + require.NoError(t, os.Mkdir(staging, 0o755)) + + stop := make(chan struct{}) + var wg sync.WaitGroup + swapID := 0 + // Atomically swap a file for a symlink. (Using rename makes this atomic; if we were to do a + // remove-then-replace, we'd eventually get unlucky and the file walker would read a file during + // replacement, returning a file-missing error unrelated to the race behavior we're trying to + // elicit.) + swapFile := func(target string) { + stageSym := filepath.Join(staging, fmt.Sprintf("sym-%d", swapID)) + stageReg := filepath.Join(staging, fmt.Sprintf("reg-%d", swapID)) + restore := []byte("restored:" + target) + swapID++ + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + _ = os.Symlink(fileswapPath, stageSym) + _ = os.Rename(stageSym, target) + _ = os.WriteFile(stageReg, restore, 0o644) + _ = os.Rename(stageReg, target) + } + }() + } + + mkTree := func(name string, count, swap int) string { + dir := filepath.Join(roundRoot, name) + require.NoError(t, os.Mkdir(dir, 0o755)) + for i := range count { + path := filepath.Join(dir, fmt.Sprintf("f%04d", i)) + require.NoError(t, os.WriteFile(path, fmt.Appendf([]byte{}, "benign-%s-%d-%d", name, round, i), 0o644)) + if i < swap { + swapFile(path) + } + } + return dir + } + + publicDir := mkTree("public", 60, 30) + edgeRedirectsDir := mkTree("edge-redirects", 60, 30) + migrationsDir := mkTree("db-migrations", 60, 30) + + functionsDir := filepath.Join(roundRoot, "functions") + require.NoError(t, os.Mkdir(functionsDir, 0o755)) + var fnEntries []map[string]any + for i := range 25 { + p := filepath.Join(functionsDir, fmt.Sprintf("fn%04d.tar", i)) + require.NoError(t, os.WriteFile(p, fmt.Appendf([]byte{}, "benign-fn-%d-%d", round, i), 0o644)) + swapFile(p) + fnEntries = append(fnEntries, map[string]any{ + "name": fmt.Sprintf("fn%04d", i), + "path": p, + "runtime": "go", + }) + } + fnManifest, err := json.Marshal(map[string]any{"functions": fnEntries, "version": 1}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(functionsDir, "manifest.json"), fnManifest, 0o644)) + + edgeFunctionsDir := filepath.Join(roundRoot, "edge-functions") + require.NoError(t, os.Mkdir(edgeFunctionsDir, 0o755)) + var edgeBundles []map[string]string + for i := range 12 { + asset := fmt.Sprintf("asset%04d.tar.gz", i) + p := filepath.Join(edgeFunctionsDir, asset) + require.NoError(t, os.WriteFile(p, fmt.Appendf([]byte{}, "benign-edge-%d-%d", round, i), 0o644)) + swapFile(p) + edgeBundles = append(edgeBundles, map[string]string{"asset": asset, "format": fmt.Sprintf("fmt%02d", i)}) + } + for i := range 30 { + p := filepath.Join(edgeFunctionsDir, fmt.Sprintf("extra%04d.txt", i)) + require.NoError(t, os.WriteFile(p, fmt.Appendf([]byte{}, "benign-edgeextra-%d-%d", round, i), 0o644)) + swapFile(p) + } + edgeManifest, err := json.Marshal(map[string]any{"bundles": edgeBundles}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(edgeFunctionsDir, "manifest.json"), edgeManifest, 0o644)) + + opts := &porcelain.DeployOptions{ + SiteID: "site-1", + Dir: publicDir, + FunctionsDir: functionsDir, + EdgeFunctionsDir: edgeFunctionsDir, + EdgeRedirectsDir: edgeRedirectsDir, + DbMigrationsDir: migrationsDir, + UploadTimeout: time.Minute, + } + + // It's OK to return an error here, which is expected if DoDeploy rejects files altered after + // the deploy begins. (We assert below that no symlinked data was uploaded.) + // nolint:errcheck + client.DoDeploy(ctx, opts, nil) + + close(stop) + wg.Wait() + } + + fileswapsMu.Lock() + defer fileswapsMu.Unlock() + if len(fileswaps) > 0 { + labels := make([]string, 0, len(fileswaps)) + for k := range fileswaps { + labels = append(labels, k) + } + sort.Strings(labels) + var b strings.Builder + for _, k := range labels { + fmt.Fprintf(&b, "\n - %s (x%d)", k, fileswaps[k]) + } + t.Fatalf("DoDeploy: contents changed during deploy: %d branch/window combination(s):%s", + len(labels), b.String()) + } +} + +func TestDoDeploy_SymlinkAncestorTOCTOURace(t *testing.T) { + const deployID = "deploy-1" + root := t.TempDir() + + outsideContents := []byte("OUTSIDE-TREE-CONTENTS" + strings.Repeat("A", 4096)) + outsideDigest := sha1.Sum(outsideContents) + outsideSHA1 := hex.EncodeToString(outsideDigest[:]) + + var fileswapsMu sync.Mutex + fileswaps := 0 + recordFileswap := func() { + fileswapsMu.Lock() + defer fileswapsMu.Unlock() + fileswaps++ + } + + mux := http.NewServeMux() + mux.HandleFunc("POST /api/v1/sites/{site_id}/deploys", func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var data struct { + Files map[string]string `json:"files"` + } + _ = json.Unmarshal(body, &data) + for _, sum := range data.Files { + if strings.HasPrefix(sum, outsideSHA1) { + recordFileswap() + } + } + // Require nothing so DoDeploy returns before uploading + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": deployID, + "site_id": r.PathValue("site_id"), + "state": "ready", + }) + }) + server := httptest.NewServer(mux) + defer server.Close() + + hu, err := url.Parse(server.URL) + require.NoError(t, err) + tr := apiClient.NewWithClient(hu.Host, "/api/v1", []string{"http"}, http.DefaultClient) + client := porcelain.NewRetryable(tr, strfmt.Default, 1) + client.SetSyncFileLimit(1_000_000) + ctx := context.WithAuthInfo(gocontext.Background(), apiClient.BearerToken("token")) + + for round := range 6 { + roundRoot := filepath.Join(root, fmt.Sprintf("round-%d", round)) + require.NoError(t, os.MkdirAll(roundRoot, 0o755)) + + publicDir := filepath.Join(roundRoot, "public") + require.NoError(t, os.Mkdir(publicDir, 0o755)) + + decoy := filepath.Join(roundRoot, "decoy") + require.NoError(t, os.Mkdir(decoy, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(decoy, "f"), outsideContents, 0o644)) + + stop := make(chan struct{}) + var wg sync.WaitGroup + + for i := range 40 { + // A real directory holding a benign file the walk records, and an alternate symlink to the + // decoy. Both live in the same parent so their names can be exchanged atomically. + realDir := filepath.Join(publicDir, fmt.Sprintf("d%03d", i)) + require.NoError(t, os.Mkdir(realDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(realDir, "f"), fmt.Appendf([]byte{}, "benign-%d-%d", round, i), 0o644)) + altLink := filepath.Join(publicDir, fmt.Sprintf("d%03d-alt", i)) + require.NoError(t, os.Symlink(decoy, altLink)) + + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + // Atomically replace the directory with a symlink. (Rename makes it atomic; a nonatomic + // version would make the directory momentarily disappear, which could result in an + // unrelated ENOENT error instead.) + _ = unix.Renameat2(unix.AT_FDCWD, realDir, unix.AT_FDCWD, altLink, unix.RENAME_EXCHANGE) + } + }() + } + + opts := &porcelain.DeployOptions{ + SiteID: "site-1", + Dir: publicDir, + } + // We can ignore errors here--this test is only concerned with checking if DoDeploy read files + // outside of the build directory root. + // nolint:errcheck + client.DoDeploy(ctx, opts, nil) + + close(stop) + wg.Wait() + } + + fileswapsMu.Lock() + defer fileswapsMu.Unlock() + if fileswaps > 0 { + t.Fatalf("DoDeploy hashed out-of-tree contents through a symlinked parent directory %d time(s)", fileswaps) + } +} diff --git a/go/porcelain/deploy_unix.go b/go/porcelain/deploy_other.go similarity index 80% rename from go/porcelain/deploy_unix.go rename to go/porcelain/deploy_other.go index 191808b18..c78a407c6 100644 --- a/go/porcelain/deploy_unix.go +++ b/go/porcelain/deploy_other.go @@ -1,4 +1,4 @@ -// +build !windows +//go:build !windows package porcelain diff --git a/go/porcelain/deploy_test.go b/go/porcelain/deploy_test.go index 7887cbe9b..25f1a6000 100644 --- a/go/porcelain/deploy_test.go +++ b/go/porcelain/deploy_test.go @@ -1013,7 +1013,7 @@ func TestBundleWithManifestEventSubscriptions(t *testing.T) { } func TestReadZipRuntime(t *testing.T) { - runtime, err := readZipRuntime("../internal/data/hello-rs-function-test.zip") + runtime, err := readZipRuntime("../internal/data", "../internal/data/hello-rs-function-test.zip") assert.Nil(t, err) assert.Equal(t, "rs", runtime) diff --git a/go/porcelain/deploy_windows.go b/go/porcelain/deploy_windows.go index 279c05bee..02d280d39 100644 --- a/go/porcelain/deploy_windows.go +++ b/go/porcelain/deploy_windows.go @@ -1,3 +1,5 @@ +//go:build windows + package porcelain import (