Add automatic build retention

docs/architecture.md

@@ -69,6 +69,7 @@ src/
 ```json
 {
   "pollIntervalSeconds": 300,
+  "backfillDays": 14,
   "sources": [
     {
       "organization": "dnceng-public",
@@ -84,6 +85,10 @@ src/
 }
 ```
 
+`backfillDays` is the single build-data window: backfills fetch this many days of builds,
+and a background retention service runs at startup and every 15 minutes, deleting builds
+older than the same window and their associated database rows.
+
 ## SQLite Schema (Multi-Org Aware)
 
 Build IDs are unique within an organization in Azure DevOps.

src/Tiger.Tests/BuildRetentionServiceTests.cs

@@ -0,0 +1,66 @@
+using Xunit;
+
+namespace Tiger.Tests;
+
+public class BuildRetentionServiceTests : IDisposable
+{
+    private readonly string _dbPath;
+    private readonly TigerDatabase _db;
+
+    public BuildRetentionServiceTests()
+    {
+        _dbPath = Path.Combine(Path.GetTempPath(), $"tiger-retention-test-{Guid.NewGuid()}.db");
+        _db = TigerDatabase.Open(_dbPath);
+    }
+
+    public void Dispose()
+    {
+        _db.Dispose();
+        if (File.Exists(_dbPath))
+        {
+            File.Delete(_dbPath);
+        }
+    }
+
+    [Fact]
+    public void CleanupExpiredBuilds_DeletesBuildsPastRetentionAcrossOrganizations()
+    {
+        var oldFinishTime = DateTime.UtcNow.AddDays(-15).ToString("o");
+        var recentFinishTime = DateTime.UtcNow.AddDays(-10).ToString("o");
+        _db.WithCommand(cmd =>
+        {
+            cmd.CommandText = """
+                INSERT INTO builds
+                    (organization, project, build_id, build_number, definition_name, definition_id, status, source_branch, finish_time)
+                VALUES
+                    ('org1', 'proj', 1, 'old-1', 'pipeline', 1, 'completed', 'main', @old),
+                    ('org2', 'proj', 1, 'old-2', 'pipeline', 1, 'completed', 'main', @old),
+                    ('org1', 'proj', 2, 'recent', 'pipeline', 1, 'completed', 'main', @recent),
+                    ('org1', 'proj', 3, 'unknown-time', 'pipeline', 1, 'completed', 'main', NULL)
+                """;
+            cmd.Parameters.AddWithValue("@old", oldFinishTime);
+            cmd.Parameters.AddWithValue("@recent", recentFinishTime);
+            cmd.ExecuteNonQuery();
+        });
+
+        var service = new BuildRetentionService(new TigerConfig { BackfillDays = 14 }, _db);
+        var deleted = service.CleanupExpiredBuilds();
+
+        Assert.Equal(2, deleted);
+        var remainingBuilds = _db.WithCommand(cmd =>
+        {
+            cmd.CommandText = "SELECT organization || ':' || build_id FROM builds ORDER BY organization, build_id";
+            using var reader = cmd.ExecuteReader();
+            var builds = new List<string>();
+            while (reader.Read())
+            {
+                builds.Add(reader.GetString(0));
+            }
+            return string.Join("\n", builds);
+        });
+        Assert.Equal("""
+            org1:2
+            org1:3
+            """.ReplaceLineEndings("\n"), remainingBuilds);
+    }
+}

src/Tiger.Tests/TigerConfigTests.cs

@@ -0,0 +1,16 @@
+using Xunit;
+
+namespace Tiger.Tests;
+
+public class TigerConfigTests
+{
+    [Fact]
+    public void BackfillDays_DefaultsToFourteen()
+    {
+        var configDirectory = Path.Combine(Path.GetTempPath(), $"tiger-config-test-{Guid.NewGuid()}");
+
+        var config = TigerConfig.Load(configDirectory);
+        Assert.Equal(14, new TigerConfig().BackfillDays);
+        Assert.Equal(14, config.BackfillDays);
+    }
+}

src/Tiger/BuildRetentionService.cs

@@ -0,0 +1,90 @@
+namespace Tiger;
+
+/// <summary>
+/// Periodically removes builds and their associated data after the configured retention period.
+/// </summary>
+public sealed class BuildRetentionService : IDisposable
+{
+    private static readonly TimeSpan s_cleanupInterval = TimeSpan.FromMinutes(15);
+
+    private readonly TigerConfig _config;
+    private readonly TigerDatabase _db;
+    private readonly ServiceLog? _log;
+    private CancellationTokenSource? _cts;
+    private Task _cleanupTask = Task.CompletedTask;
+
+    public bool IsRunning => !_cleanupTask.IsCompleted;
+
+    public BuildRetentionService(TigerConfig config, TigerDatabase db, ServiceLog? log = null)
+    {
+        _config = config;
+        _db = db;
+        _log = log;
+    }
+
+    public void Start()
+    {
+        if (IsRunning)
+        {
+            return;
+        }
+
+        _cts = new CancellationTokenSource();
+        _cleanupTask = CleanupLoopAsync(_cts.Token);
+    }
+
+    public async Task StopAsync()
+    {
+        if (_cts is not null)
+        {
+            await _cts.CancelAsync();
+            try
+            {
+                await _cleanupTask;
+            }
+            catch (OperationCanceledException)
+            {
+            }
+            _cts.Dispose();
+            _cts = null;
+            _cleanupTask = Task.CompletedTask;
+        }
+    }
+
+    private async Task CleanupLoopAsync(CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                var deletedCount = CleanupExpiredBuilds();
+                _log?.Info("Retention", $"Deleted {deletedCount} build(s) older than {_config.BackfillDays} days.");
+            }
+            catch (Exception ex) when (ex is not OperationCanceledException)
+            {
+                _log?.Error("Retention", $"Failed to clean up expired builds: {ex.Message}");
+            }
+
+            try
+            {
+                await Task.Delay(s_cleanupInterval, ct);
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+        }
+    }
+
+    internal int CleanupExpiredBuilds()
+    {
+        var cutoff = DateTime.UtcNow.AddDays(-_config.BackfillDays);
+        return _db.DeleteBuildsOlderThan(cutoff);
+    }
+
+    public void Dispose()
+    {
+        _cts?.Cancel();
+        _cts?.Dispose();
+    }
+}

src/Tiger/Commands/ConfigEditor.cs

@@ -84,7 +84,7 @@ private void RenderConfig()
         table.AddColumn("Setting");
         table.AddColumn("Value");
         table.AddRow("Poll interval", $"{_config.PollIntervalSeconds}s");
-        table.AddRow("Backfill days", _config.BackfillDays.ToString());
+        table.AddRow("Ingestion window", $"{_config.BackfillDays} days");
         AnsiConsole.Write(table);
         AnsiConsole.WriteLine();
 

src/Tiger/Commands/DashboardCommand.cs

@@ -51,6 +51,9 @@ protected override async Task<int> ExecuteAsync(CommandContext context, Cancella
         var healthAgent = new HealthAgentService(config, db, serviceLog);
         healthAgent.Start();
 
+        var retention = new BuildRetentionService(config, db, serviceLog);
+        retention.Start();
+
         var backfill = new BuildBackfillService(config, db, ingestion, clientFactory, serviceLog);
         backfill.Start();
 
@@ -64,6 +67,7 @@ protected override async Task<int> ExecuteAsync(CommandContext context, Cancella
             await ingestion.StopAsync();
             await poller.StopAsync();
             await backfill.StopAsync();
+            await retention.StopAsync();
             await analysisAgent.StopAsync();
         }
 
@@ -266,4 +270,3 @@ void OnEntry()
         }
     }
 }
-

src/Tiger/Commands/PollerCommands.cs

@@ -24,6 +24,7 @@ protected override async Task<int> ExecuteAsync(CommandContext context, Cancella
         var ingestion = new BuildIngestionService(db);
         var clientFactory = new AzdoClientFactory(tigerContext.AzureCredential);
         var poller = new BuildPoller(config, db, clientFactory);
+        var retention = new BuildRetentionService(config, db);
 
         poller.OnNewBuilds = ingestion.InsertBuildsAsync;
 
@@ -38,6 +39,7 @@ protected override async Task<int> ExecuteAsync(CommandContext context, Cancella
         AnsiConsole.WriteLine();
 
         poller.Start();
+        retention.Start();
 
         try
         {
@@ -49,6 +51,7 @@ protected override async Task<int> ExecuteAsync(CommandContext context, Cancella
 
         AnsiConsole.MarkupLine("[yellow]Stopping poller...[/]");
         await poller.StopAsync();
+        await retention.StopAsync();
         AnsiConsole.MarkupLine("[green]Poller stopped.[/]");
 
         return 0;

src/Tiger/TigerConfig.cs

@@ -70,7 +70,7 @@ public sealed class TigerConfig
     public int PollIntervalSeconds { get; set; } = 300;
 
     [JsonPropertyName("backfillDays")]
-    public int BackfillDays { get; set; } = 7;
+    public int BackfillDays { get; set; } = 14;
 
     [JsonPropertyName("sources")]
     public List<AzdoSource> Sources { get; set; } = [];
@@ -111,6 +111,11 @@ public static string GetConfigPath(string configDirectory) =>
 
     private void Normalize()
     {
+        if (BackfillDays <= 0)
+        {
+            throw new InvalidOperationException("Backfill days must be greater than zero.");
+        }
+
         foreach (var source in Sources)
         {
             source.RepositoryType = AzdoRepositoryTypes.Normalize(source.RepositoryType);
@@ -129,6 +134,6 @@ private void Normalize()
                 Repositories = ["dotnet/roslyn"],
             }
         ],
-        BackfillDays = 3,
+        BackfillDays = 14,
     };
 }

src/Tiger/TigerDatabase.cs

@@ -498,6 +498,35 @@ DELETE FROM builds
         });
     }
 
+    public int DeleteBuildsOlderThan(DateTime cutoff)
+    {
+        var expiredBuilds = WithCommand(cmd =>
+        {
+            cmd.CommandText = """
+                SELECT organization, build_id
+                FROM builds
+                WHERE finish_time IS NOT NULL
+                  AND datetime(finish_time) < datetime(@cutoff)
+                """;
+            cmd.Parameters.AddWithValue("@cutoff", cutoff.ToUniversalTime().ToString("o"));
+
+            var builds = new List<(string Organization, int BuildId)>();
+            using var reader = cmd.ExecuteReader();
+            while (reader.Read())
+            {
+                builds.Add((reader.GetString(0), reader.GetInt32(1)));
+            }
+            return builds;
+        });
+
+        foreach (var (organization, buildId) in expiredBuilds)
+        {
+            DeleteBuild(organization, buildId);
+        }
+
+        return expiredBuilds.Count;
+    }
+
     /// <summary>
     /// Inserts a new build analysis row with status 'pending'.
     /// </summary>

src/Tiger/skills/tiger-data/SKILL.md

@@ -279,6 +279,7 @@ ORDER BY b.finish_time DESC;
 - `pull_requests`, `known_issues`, and `helix_work_items` are keyed by repository or job (not org/project)
 - Times are stored as ISO 8601 strings in UTC
 - Only failed test results are stored (not passing tests)
+- Build records and their associated test, timeline, Helix, and analysis rows are automatically deleted after the configured `backfillDays` period (14 days by default). Cleanup runs at startup and every 15 minutes. The same setting controls how far back backfill fetches builds.
 - The `builds.ingestion_status` column shows whether test/timeline/Helix data is available for a build (tests, timeline, and Helix work items are ingested together in a single atomic pass)
 - Helix work items are fetched inline as part of build ingestion (no separate helix task)
 - Known issues are refreshed every 15 minutes; closed issues are kept for 7 days before purging