Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
target/
*.orig
.DS_Store

# local secrets / environment — never in the repo
.env
Expand Down
64 changes: 43 additions & 21 deletions crates/walgit-config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,8 @@ pub struct Config {
#[serde(deny_unknown_fields, default)]
pub struct ServerConfig {
pub listen: SocketAddr,
pub http2: bool,
pub max_concurrent_requests: usize,
/// Per-repo cap on concurrent upload-pack/receive-pack processes.
pub max_concurrent_per_repo: usize,
#[serde(with = "humantime_serde")]
pub request_timeout: Duration,
/// Graceful drain after SIGTERM: new object work (fetch/push/LFS) is
/// refused with 503 + Retry-After and `/readyz` turns 503 at once; in-flight
/// requests and the running maintenance unit get this long to finish
Expand Down Expand Up @@ -268,7 +264,6 @@ pub enum StoreBackend {
pub struct GcsConfig {
/// gRPC endpoint.
pub endpoint: String,
pub direct_connectivity: bool,
/// Service account for signed URLs; None = ADC/IAM signBlob.
pub signing_service_account: Option<String>,
/// Separate data clients (own channels) for bulk traffic — pack/idx/side-
Expand Down Expand Up @@ -584,11 +579,8 @@ pub struct UpstreamConfig {
reason = "Independent configuration switches, not mutually exclusive states"
)]
pub struct GitConfig {
/// Path to the upstream git binary (repack, optional upload-pack engine).
pub binary: PathBuf,
pub upload_pack_engine: UploadPackEngine,
pub allow_filter: bool,
pub allow_any_sha1_in_want: bool,
/// Default object format for new repos.
pub object_format: ObjectFormat,
/// Maintain a split commit-graph chain per local repo: tier-2 packs that
Expand Down Expand Up @@ -863,10 +855,7 @@ impl Default for ServerConfig {
fn default() -> Self {
ServerConfig {
listen: std::net::SocketAddr::from(([127, 0, 0, 1], 8080)),
http2: true,
max_concurrent_requests: 512,
max_concurrent_per_repo: 64,
request_timeout: Duration::from_hours(1),
drain_timeout: Duration::from_secs(20),
max_push_bytes: ByteSize::gib(64),
roles: vec![],
Expand Down Expand Up @@ -919,7 +908,6 @@ impl Default for GcsConfig {
fn default() -> Self {
GcsConfig {
endpoint: "https://storage.googleapis.com".into(),
direct_connectivity: true,
signing_service_account: None,
bulk_clients: 4,
bulk_concurrency: 32,
Expand Down Expand Up @@ -991,10 +979,8 @@ impl Default for LfsConfig {
impl Default for GitConfig {
fn default() -> Self {
GitConfig {
binary: PathBuf::from("git"),
upload_pack_engine: UploadPackEngine::Auto,
allow_filter: true,
allow_any_sha1_in_want: false,
object_format: ObjectFormat::Sha1,
commit_graph: true,
commit_graph_changed_paths: false,
Expand Down Expand Up @@ -1503,10 +1489,12 @@ mod tests {
// `Config::parse` (startup) reads the process env through the same path.
let mut c = Config::default();
assert!(
c.apply_env(vec![("WALGIT__GIT__BINARI".to_string(), "g".to_string())].into_iter())
.unwrap_err()
.to_string()
.contains("did you mean WALGIT__GIT__BINARY?")
c.apply_env(
vec![("WALGIT__GIT__HISTORY_PAC".to_string(), "g".to_string())].into_iter()
)
.unwrap_err()
.to_string()
.contains("did you mean WALGIT__GIT__HISTORY_PACK?")
);
}

Expand Down Expand Up @@ -1535,17 +1523,23 @@ mod tests {
let err = c
.apply_env(
vec![
("WALGIT__SERVER__HTTP2".to_string(), "1".to_string()),
(
"WALGIT__SERVER__ACCEL_REDIRECT".to_string(),
"1".to_string(),
),
("WALGIT__STORE__BUCKET__NAME".to_string(), "b".to_string()),
]
.into_iter(),
)
.unwrap_err()
.to_string();
assert!(err.contains("WALGIT__SERVER__HTTP2: "), "{err}");
assert!(err.contains("WALGIT__SERVER__ACCEL_REDIRECT: "), "{err}");
assert!(err.contains("WALGIT__STORE__BUCKET__NAME: "), "{err}");
assert!(err.contains("not a table"), "{err}");
assert_eq!(c.server.http2, Config::default().server.http2);
assert_eq!(
c.server.accel_redirect,
Config::default().server.accel_redirect
);
assert_eq!(c.store.bucket, "20260915");
}

Expand All @@ -1564,6 +1558,17 @@ mod tests {
assert!(!base.public_settings_toml().unwrap().contains("[bundles]"));
}

#[test]
fn removed_global_request_limit_configuration_is_rejected() {
for input in [
"[server]\nmax_concurrent_requests = 512\n",
"[server]\nrequest_timeout = \"1h\"\n",
] {
assert!(Config::parse(input).is_err(), "{input}");
assert!(Config::default().with_settings(input).is_err(), "{input}");
}
}

#[test]
fn removed_compaction_configuration_is_rejected() {
for input in [
Expand All @@ -1584,6 +1589,23 @@ mod tests {
);
}

#[test]
fn removed_git_binary_configuration_is_rejected() {
let input = "[git]\nbinary = \"/opt/git/bin/git\"\n";
assert!(Config::parse(input).is_err());
assert!(Config::default().with_settings(input).is_err());
}

#[test]
fn removed_transport_switches_are_rejected() {
for input in [
"[server]\nhttp2 = false\n",
"[store.gcs]\ndirect_connectivity = false\n",
] {
assert!(Config::parse(input).is_err(), "{input}");
}
}

#[test]
fn settings_merge_over_config_and_are_restricted() {
let mut base = Config::default();
Expand Down
6 changes: 2 additions & 4 deletions crates/walgit-server/src/middleware.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
//! HTTP middleware: request id + tracing span. Request timeout, body limit and
//! tracing layers are applied in [`crate::router`] via `tower-http`. Per-repo
//! concurrency limiting lives in the handlers (they hold a repo-keyed semaphore
//! for the duration of the git operation).
//! HTTP middleware: request id + tracing span and per-repository concurrency
//! limits. Per-repository permits live in handlers for the full Git operation.

use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
Expand Down
2 changes: 1 addition & 1 deletion crates/walgit-server/tests/e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1653,7 +1653,7 @@ async fn push_from_a_shallow_clone() -> TestResult {
/// `allow-any-sha1-in-want`), plus `--depth` + filter together.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn partial_clone_tree_zero_and_depth_with_filter() -> TestResult {
let server = Server::start_with_tweak(|c| c.git.allow_any_sha1_in_want = true).await?;
let server = Server::start().await?;
server.put_repo("t", "tree0").await?;
let src = TestRepo::synthetic(5, 4)?;
git_in(
Expand Down
1 change: 0 additions & 1 deletion crates/walgit-server/tests/harness.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ impl Server {
cfg.cache.max_bytes = ByteSize::gib(2);
cfg.server.listen = "127.0.0.1:0".parse().unwrap();
cfg.server.max_concurrent_per_repo = 8;
cfg.server.request_timeout = std::time::Duration::from_mins(10);
cfg.server.max_push_bytes = ByteSize::gib(2);
cfg.wal.fsck_objects = true;
cfg.wal.check_connectivity = true;
Expand Down
1 change: 0 additions & 1 deletion crates/walgit-server/tests/maintain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,6 @@ async fn fsck_unit_records_missing_objects_and_repair_unit_fetches_them_from_ups
let server = step!(
"start",
Server::start_with_tweak(|c| {
c.git.allow_any_sha1_in_want = true;
c.maintenance.checkpoints = false;
c.packs.enabled = false;
c.maintenance.fsck_interval = std::time::Duration::from_hours(1);
Expand Down
6 changes: 5 additions & 1 deletion docs/INTEGRITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,18 @@ Due right after checkpoint when `fsck.pb` lists missing objects, `repaired_seq =
**`upstream.git`** (D24 setting, `[upstream] git = "https://github.com/acme/monorepo.git"`, `token_env` shared with
`upstream.lfs`, `docs/LFS.md`). Steps (`walgit_git::repair::fetch_objects_as_pack`): scratch bare repo under
`cache.dir/repair/`, `git fetch --depth=1 <upstream> <oid>…` in batches of 500 (GitHub serves commit, tree **and
blob** wants by SHA — verified 2026-08-21; walgit does with `git.allow_any_sha1_in_want`), `pack-objects` of exactly
blob** wants by SHA — verified 2026-08-21; walgit enables arbitrary object wants for partial-clone lazy fetches), `pack-objects` of exactly
the requested oids, verify every oid is in the resulting idx (a refused want is an error, never a silent hole),
then `RepoHandle::add_pack(tier 0)` → one **COMPACT entry superseding nothing** (what `walgit wal add-pack … --tier
0` did by hand for a large repository, seq 11). `fsck.pb.repaired_seq` is set so the next pass re-audits instead of repairing
again. Counter `walgit_repair_objects_total{repo}`. Test: `tests/maintain.rs
fsck_unit_records_missing_objects_and_repair_unit_fetches_them_from_upstream` (hole → audit → repair over HTTP from
a second repository → re-audit clean → idle).

This does not widen repository access: a principal with read access can already fetch any stored object by ID,
including objects unreachable from an advertised ref. Removing `git.allow_any_sha1_in_want` only removes a
non-functional configuration switch.

## 4. Runbook (what was done for a large repository, 2026-08-21 03:40Z)
1. Enumerate: `git rev-list --objects --missing=print <advertised tips> --not <known-good tip>` on the complete copy
(or read `fsck.pb` / `/var/lib/walgit/monorepo-fsck.out`) → 1,952 blobs, 70 MB.
Expand Down
6 changes: 0 additions & 6 deletions walgit.example.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,7 @@

[server]
listen = "127.0.0.1:8080" # default; `mode = none` is refused unless this is loopback. Public bind: 0.0.0.0 with token/oidc.
http2 = true # HTTP/2 (h2c when TLS is off, ALPN when on)
max_concurrent_requests = 512 # global cap on in-flight git requests
max_concurrent_per_repo = 64 # per-repo cap (upload-pack / receive-pack)
request_timeout = "1h"
drain_timeout = "20s" # after SIGTERM, phase 2: how long in-flight requests may finish once /readyz is 503
# and new fetch/push/LFS are refused (503 + Retry-After). The running maintenance unit
# is interrupted at once in phase 1 while serving continues.
Expand Down Expand Up @@ -84,7 +81,6 @@ force_path_style = true # path-style `endpoint/bucket/key`, what rustf

[store.gcs]
endpoint = "https://storage.googleapis.com" # gRPC endpoint; credentials = Application Default Credentials
direct_connectivity = true
# signing_service_account = "sa@project.iam.gserviceaccount.com" # for lfs serve_via = "signed_url"
bulk_clients = 4 # separate data channels for pack/idx/side-file/LFS bytes + ranged reads
bulk_concurrency = 32 # max concurrent bulk requests per process (control plane never queues behind them)
Expand Down Expand Up @@ -183,10 +179,8 @@ max_object_bytes = "16GiB"
# maintenance.follow_interval, published through the WAL as a push; fast-forward only

[git]
binary = "git" # path to upstream git binary (>= 2.47 recommended: pack.writeReverseIndex)
upload_pack_engine = "auto" # "auto" = stock git when packs are local/mount-linked, gix only for remote-served bases | "git" | "gix"
allow_filter = true # uploadpack.allowFilter
allow_any_sha1_in_want = false # uploadpack.allowAnySHA1InWant
object_format = "sha1" # default for new repos: "sha1" | "sha256"
commit_graph = true # maintain a split commit-graph chain per repo (tier-2 layer from the WAL + incremental)
history_pack = true # base rebuild also publishes commits+trees as a local "history pack"
Expand Down
Loading