From b861161421e8c3124c3eaedbd2fc1f28a6511be1 Mon Sep 17 00:00:00 2001 From: Charlie Morrison Date: Sun, 8 Nov 2026 15:45:20 +0200 Subject: [PATCH 1/3] Decode git-config value syntax in .gitmodules Fixes #130 --- internal/git/git.go | 75 ++++++++++++++++++++++++++++++++++++++-- internal/git/git_test.go | 48 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 3 deletions(-) diff --git a/internal/git/git.go b/internal/git/git.go index 92c9229..726d010 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -30,8 +30,8 @@ func (p *gitmodulesParser) Parse(filename string, content []byte) (*core.Result, lines := strings.Split(string(content), "\n") var current *submodule - for _, line := range lines { - line = strings.TrimSpace(line) + for i := 0; i < len(lines); i++ { + line := strings.TrimSpace(lines[i]) if match := submoduleHeaderRegex.FindStringSubmatch(line); match != nil { if current != nil { @@ -45,7 +45,20 @@ func (p *gitmodulesParser) Parse(filename string, content []byte) (*core.Result, continue } - key, val, ok := parseKeyValue(line) + key, raw, ok := parseKeyValue(line) + if !ok { + continue + } + val, more, ok := configValue(raw) + for more { + if i+1 < len(lines) { + i++ + raw += "\n" + strings.TrimSuffix(lines[i], "\r") + } else { + raw += "\n" + } + val, more, ok = configValue(raw) + } if !ok { continue } @@ -96,3 +109,59 @@ func parseKeyValue(line string) (key, value string, ok bool) { } return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), true } + +// configValue decodes a value as git's parse_value does; more means it continues on the next line, !ok that git rejects it. +func configValue(s string) (value string, more, ok bool) { + var b strings.Builder + quote := false + trim := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if !quote && strings.IndexByte(" \t\v\f\r", c) >= 0 { + if trim == 0 { + trim = b.Len() + } + if b.Len() > 0 { + b.WriteByte(c) + } + continue + } + if !quote && (c == ';' || c == '#') { + break + } + trim = 0 + switch c { + case '\\': + i++ + if i == len(s) { + return "", true, true + } + switch s[i] { + case '\n': + continue + case 't': + b.WriteByte('\t') + case 'b': + b.WriteByte('\b') + case 'n': + b.WriteByte('\n') + case '\\', '"': + b.WriteByte(s[i]) + default: + return "", false, false + } + case '"': + quote = !quote + default: + b.WriteByte(c) + } + } + if quote { + return "", false, false + } + value = b.String() + if trim > 0 { + value = value[:trim] + } + return value, false, true +} diff --git a/internal/git/git_test.go b/internal/git/git_test.go index e114d05..58b98e1 100644 --- a/internal/git/git_test.go +++ b/internal/git/git_test.go @@ -51,3 +51,51 @@ func TestGitmodulesWithoutURL(t *testing.T) { t.Errorf("got %+v, want %+v", res.Dependencies, want) } } + +func TestGitmodulesQuotedValues(t *testing.T) { + content := "[submodule \"lib/foo\"]\n" + + "\tpath = \"lib/foo\"\n" + + "\turl = \"https://github.com/example/foo.git\" ; mirror\n" + + "\tbranch = \"fix#1\" # pinned\n" + + "[submodule \"bar\"]\r\n" + + "\tpath = bar\r\n" + + "\turl = https://github.com/example/\\\r\n" + + "bar.git\r\n" + res, err := (&gitmodulesParser{}).Parse(".gitmodules", []byte(content)) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + want := []core.Dependency{ + {Name: "github.com/example/foo", Scope: core.Runtime, Direct: true, + Source: core.Source{Kind: core.SourceGit, Value: "https://github.com/example/foo.git", Branch: "fix#1"}}, + {Name: "github.com/example/bar", Scope: core.Runtime, Direct: true, + Source: core.Source{Kind: core.SourceGit, Value: "https://github.com/example/bar.git"}}, + } + if !reflect.DeepEqual(res.Dependencies, want) { + t.Errorf("got %+v, want %+v", res.Dependencies, want) + } +} + +func TestConfigValue(t *testing.T) { + tests := []struct { + in, want string + more, ok bool + }{ + {`plain`, "plain", false, true}, + {`"quoted value"`, "quoted value", false, true}, + {`a "b c" d`, "a b c d", false, true}, + {`trailing ; comment`, "trailing", false, true}, + {`"kept " # comment`, "kept ", false, true}, + {`esc\"aped\\ \t\n`, "esc\"aped\\ \t\n", false, true}, + {`"open`, "", false, false}, + {`bad\q`, "", false, false}, + {`ends with \`, "", true, true}, + {`# only a comment \`, "", false, true}, + } + for _, tt := range tests { + got, more, ok := configValue(tt.in) + if got != tt.want || more != tt.more || ok != tt.ok { + t.Errorf("configValue(%q) = %q, %v, %v; want %q, %v, %v", tt.in, got, more, ok, tt.want, tt.more, tt.ok) + } + } +} From 9d5a1c1ac6632b3e66b1608c25779654af416c71 Mon Sep 17 00:00:00 2001 From: Charlie Morrison Date: Fri, 13 Nov 2026 17:09:57 +0200 Subject: [PATCH 2/3] Skip comment lines and keep trailing space for escape checks A comment ending in a backslash no longer continues into the next section header, and 'url = x\ ' is rejected as git does instead of joining the next line. --- internal/git/git.go | 10 +++++++--- internal/git/git_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/internal/git/git.go b/internal/git/git.go index 726d010..bbb2012 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -31,7 +31,11 @@ func (p *gitmodulesParser) Parse(filename string, content []byte) (*core.Result, var current *submodule for i := 0; i < len(lines); i++ { - line := strings.TrimSpace(lines[i]) + physical := strings.TrimSuffix(lines[i], "\r") + line := strings.TrimSpace(physical) + if strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") { + continue + } if match := submoduleHeaderRegex.FindStringSubmatch(line); match != nil { if current != nil { @@ -45,7 +49,7 @@ func (p *gitmodulesParser) Parse(filename string, content []byte) (*core.Result, continue } - key, raw, ok := parseKeyValue(line) + key, raw, ok := parseKeyValue(physical) if !ok { continue } @@ -107,7 +111,7 @@ func parseKeyValue(line string) (key, value string, ok bool) { if len(parts) != keyValueParts { return "", "", false } - return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), true + return strings.TrimSpace(parts[0]), parts[1], true } // configValue decodes a value as git's parse_value does; more means it continues on the next line, !ok that git rejects it. diff --git a/internal/git/git_test.go b/internal/git/git_test.go index 58b98e1..aca2ca6 100644 --- a/internal/git/git_test.go +++ b/internal/git/git_test.go @@ -99,3 +99,40 @@ func TestConfigValue(t *testing.T) { } } } + +func TestGitmodulesCommentLineEndingInBackslash(t *testing.T) { + content := "[submodule \"one\"]\n" + + "\tpath = one\n" + + "# example = \\\n" + + "[submodule \"two\"]\n" + + "\tpath = two\n" + + "\t; note = \\\n" + + "\turl = https://github.com/example/two.git\n" + res, err := (&gitmodulesParser{}).Parse(".gitmodules", []byte(content)) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + want := []core.Dependency{ + {Name: "one", Scope: core.Runtime, Direct: true, NoPURL: true}, + {Name: "github.com/example/two", Scope: core.Runtime, Direct: true, + Source: core.Source{Kind: core.SourceGit, Value: "https://github.com/example/two.git"}}, + } + if !reflect.DeepEqual(res.Dependencies, want) { + t.Errorf("got %+v, want %+v", res.Dependencies, want) + } +} + +func TestGitmodulesBackslashBeforeTrailingSpace(t *testing.T) { + content := "[submodule \"one\"]\n" + + "\tpath = one\n" + + "\turl = https://github.com/example/one.git\\ \n" + + "\tbranch = main\n" + res, err := (&gitmodulesParser{}).Parse(".gitmodules", []byte(content)) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + want := []core.Dependency{{Name: "one", Scope: core.Runtime, Direct: true, NoPURL: true}} + if !reflect.DeepEqual(res.Dependencies, want) { + t.Errorf("got %+v, want %+v", res.Dependencies, want) + } +} From ec179906b5dd2461765cfbffbec49249d7dc6ace Mon Sep 17 00:00:00 2001 From: Charlie Morrison Date: Sun, 15 Nov 2026 17:25:04 +0200 Subject: [PATCH 3/3] Decode .gitmodules values in one pass Parse hands configValue the rest of the file from the start of the value, and configValue returns how many bytes it used, so a long run of continuation lines is read once instead of re-copied and re-scanned per line. Whitespace matches git's isspace (space, tab, CR), and key names are matched case-insensitively. Co-authored-by: Claude --- internal/git/git.go | 84 ++++++++++++++++++++++------------------ internal/git/git_test.go | 68 +++++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 51 deletions(-) diff --git a/internal/git/git.go b/internal/git/git.go index bbb2012..25e7d2a 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -7,8 +7,6 @@ import ( "github.com/git-pkgs/manifests/internal/core" ) -const keyValueParts = 2 // key=value - func init() { core.Register("git", core.Manifest, &gitmodulesParser{}, core.ExactMatch(".gitmodules")) } @@ -27,11 +25,13 @@ type submodule struct { func (p *gitmodulesParser) Parse(filename string, content []byte) (*core.Result, error) { var deps []core.Dependency - lines := strings.Split(string(content), "\n") + s := string(content) var current *submodule - for i := 0; i < len(lines); i++ { - physical := strings.TrimSuffix(lines[i], "\r") + for pos := 0; pos < len(s); { + start := pos + pos = lineEnd(s, start) + physical := strings.TrimSuffix(strings.TrimSuffix(s[start:pos], "\n"), "\r") line := strings.TrimSpace(physical) if strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") { continue @@ -49,20 +49,13 @@ func (p *gitmodulesParser) Parse(filename string, content []byte) (*core.Result, continue } - key, raw, ok := parseKeyValue(physical) - if !ok { + eq := strings.IndexByte(physical, '=') + if eq < 0 { continue } - val, more, ok := configValue(raw) - for more { - if i+1 < len(lines) { - i++ - raw += "\n" + strings.TrimSuffix(lines[i], "\r") - } else { - raw += "\n" - } - val, more, ok = configValue(raw) - } + key := strings.ToLower(strings.TrimSpace(physical[:eq])) + val, n, ok := configValue(s[start+eq+1:]) + pos = start + eq + 1 + n if !ok { continue } @@ -106,22 +99,33 @@ func (s *submodule) dependency() core.Dependency { return dep } -func parseKeyValue(line string) (key, value string, ok bool) { - parts := strings.SplitN(line, "=", keyValueParts) - if len(parts) != keyValueParts { - return "", "", false +// lineEnd returns the index just past the newline at or after i, or len(s). +func lineEnd(s string, i int) int { + if j := strings.IndexByte(s[i:], '\n'); j >= 0 { + return i + j + 1 } - return strings.TrimSpace(parts[0]), parts[1], true + return len(s) } -// configValue decodes a value as git's parse_value does; more means it continues on the next line, !ok that git rejects it. -func configValue(s string) (value string, more, ok bool) { +// configValue decodes the value at the start of s as git's parse_value does, in one pass; n is the bytes used through the newline that ends it, !ok means git rejects it. +func configValue(s string) (value string, n int, ok bool) { var b strings.Builder - quote := false + quote, comment := false, false trim := 0 - for i := 0; i < len(s); i++ { - c := s[i] - if !quote && strings.IndexByte(" \t\v\f\r", c) >= 0 { + for n < len(s) { + c := s[n] + n++ + if c == '\r' && n < len(s) && s[n] == '\n' { + c = '\n' + n++ + } + if c == '\n' { + break + } + if comment { + continue + } + if !quote && (c == ' ' || c == '\t' || c == '\r') { if trim == 0 { trim = b.Len() } @@ -131,16 +135,22 @@ func configValue(s string) (value string, more, ok bool) { continue } if !quote && (c == ';' || c == '#') { - break + comment = true + continue } trim = 0 switch c { case '\\': - i++ - if i == len(s) { - return "", true, true + if n == len(s) { + continue + } + e := s[n] + n++ + if e == '\r' && n < len(s) && s[n] == '\n' { + e = '\n' + n++ } - switch s[i] { + switch e { case '\n': continue case 't': @@ -150,9 +160,9 @@ func configValue(s string) (value string, more, ok bool) { case 'n': b.WriteByte('\n') case '\\', '"': - b.WriteByte(s[i]) + b.WriteByte(e) default: - return "", false, false + return "", lineEnd(s, n), false } case '"': quote = !quote @@ -161,11 +171,11 @@ func configValue(s string) (value string, more, ok bool) { } } if quote { - return "", false, false + return "", n, false } value = b.String() if trim > 0 { value = value[:trim] } - return value, false, true + return value, n, true } diff --git a/internal/git/git_test.go b/internal/git/git_test.go index aca2ca6..ffeae13 100644 --- a/internal/git/git_test.go +++ b/internal/git/git_test.go @@ -3,6 +3,7 @@ package git import ( "os" "reflect" + "strings" "testing" "github.com/git-pkgs/manifests/internal/core" @@ -79,27 +80,66 @@ func TestGitmodulesQuotedValues(t *testing.T) { func TestConfigValue(t *testing.T) { tests := []struct { in, want string - more, ok bool + n int + ok bool }{ - {`plain`, "plain", false, true}, - {`"quoted value"`, "quoted value", false, true}, - {`a "b c" d`, "a b c d", false, true}, - {`trailing ; comment`, "trailing", false, true}, - {`"kept " # comment`, "kept ", false, true}, - {`esc\"aped\\ \t\n`, "esc\"aped\\ \t\n", false, true}, - {`"open`, "", false, false}, - {`bad\q`, "", false, false}, - {`ends with \`, "", true, true}, - {`# only a comment \`, "", false, true}, + {`plain`, "plain", 5, true}, + {"plain\nnext", "plain", 6, true}, + {`"quoted value"`, "quoted value", 14, true}, + {`a "b c" d`, "a b c d", 9, true}, + {`trailing ; comment`, "trailing", 20, true}, + {`"kept " # comment`, "kept ", 19, true}, + {`esc\"aped\\ \t\n`, "esc\"aped\\ \t\n", 16, true}, + {"a\\\nb\nc", "ab", 5, true}, + {"x\\\r\ny\r\nnext", "xy", 7, true}, + {`ends with \`, "ends with ", 11, true}, + {"# comment \\\nnext", "", 12, true}, + {"a \v", "a \v", 3, true}, + {"a \f\r", "a \f", 4, true}, + {`"open`, "", 5, false}, + {"\"open\nnext", "", 6, false}, + {"bad\\q rest\nnext", "", 11, false}, } for _, tt := range tests { - got, more, ok := configValue(tt.in) - if got != tt.want || more != tt.more || ok != tt.ok { - t.Errorf("configValue(%q) = %q, %v, %v; want %q, %v, %v", tt.in, got, more, ok, tt.want, tt.more, tt.ok) + got, n, ok := configValue(tt.in) + if got != tt.want || n != tt.n || ok != tt.ok { + t.Errorf("configValue(%q) = %q, %d, %v; want %q, %d, %v", tt.in, got, n, ok, tt.want, tt.n, tt.ok) } } } +func TestGitmodulesManyContinuationLines(t *testing.T) { + const lines = 5000 + content := "[submodule \"one\"]\n\tpath = one\n\turl = https://github.com/example/one.git\n\tbranch = " + + strings.Repeat("a\\\n", lines) + "\n[submodule \"two\"]\n\tpath = two\n" + res, err := (&gitmodulesParser{}).Parse(".gitmodules", []byte(content)) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + want := []core.Dependency{{Name: "github.com/example/one", Scope: core.Runtime, Direct: true, + Source: core.Source{Kind: core.SourceGit, Value: "https://github.com/example/one.git", Branch: strings.Repeat("a", lines)}}, + {Name: "two", Scope: core.Runtime, Direct: true, NoPURL: true}} + if !reflect.DeepEqual(res.Dependencies, want) { + t.Errorf("got %d deps, want the branch joined from %d continuation lines and the submodule after it", len(res.Dependencies), lines) + } +} + +func TestGitmodulesKeysIgnoreCase(t *testing.T) { + content := "[submodule \"one\"]\n" + + "\tPath = one\n" + + "\tURL = https://github.com/example/one.git\n" + + "\tBranch = Main\n" + res, err := (&gitmodulesParser{}).Parse(".gitmodules", []byte(content)) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + want := []core.Dependency{{Name: "github.com/example/one", Scope: core.Runtime, Direct: true, + Source: core.Source{Kind: core.SourceGit, Value: "https://github.com/example/one.git", Branch: "Main"}}} + if !reflect.DeepEqual(res.Dependencies, want) { + t.Errorf("got %+v, want %+v", res.Dependencies, want) + } +} + func TestGitmodulesCommentLineEndingInBackslash(t *testing.T) { content := "[submodule \"one\"]\n" + "\tpath = one\n" +