From cc85971ac82482599c41fa55319359ac48131685 Mon Sep 17 00:00:00 2001 From: Jared Parsons Date: Tue, 29 Sep 2026 14:43:08 -0700 Subject: [PATCH] Add automatic build retention Use the ingestion window as the retention period and periodically remove expired builds and their dependent database data.\n\nCo-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/architecture.md | 5 ++ src/Tiger.Tests/BuildRetentionServiceTests.cs | 66 ++++++++++++++ src/Tiger.Tests/TigerConfigTests.cs | 16 ++++ src/Tiger/BuildRetentionService.cs | 90 +++++++++++++++++++ src/Tiger/Commands/ConfigEditor.cs | 2 +- src/Tiger/Commands/DashboardCommand.cs | 5 +- src/Tiger/Commands/PollerCommands.cs | 3 + src/Tiger/TigerConfig.cs | 9 +- src/Tiger/TigerDatabase.cs | 29 ++++++ src/Tiger/skills/tiger-data/SKILL.md | 1 + 10 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 src/Tiger.Tests/BuildRetentionServiceTests.cs create mode 100644 src/Tiger.Tests/TigerConfigTests.cs create mode 100644 src/Tiger/BuildRetentionService.cs diff --git a/docs/architecture.md b/docs/architecture.md index 5056745..65e9e5b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,6 +69,7 @@ src/ ```json { "pollIntervalSeconds": 300, + "backfillDays": 14, "sources": [ { "organization": "dnceng-public", @@ -84,6 +85,10 @@ src/ } ``` +`backfillDays` is the single build-data window: backfills fetch this many days of builds, +and a background retention service runs at startup and every 15 minutes, deleting builds +older than the same window and their associated database rows. + ## SQLite Schema (Multi-Org Aware) Build IDs are unique within an organization in Azure DevOps. diff --git a/src/Tiger.Tests/BuildRetentionServiceTests.cs b/src/Tiger.Tests/BuildRetentionServiceTests.cs new file mode 100644 index 0000000..b3fd3d5 --- /dev/null +++ b/src/Tiger.Tests/BuildRetentionServiceTests.cs @@ -0,0 +1,66 @@ +using Xunit; + +namespace Tiger.Tests; + +public class BuildRetentionServiceTests : IDisposable +{ + private readonly string _dbPath; + private readonly TigerDatabase _db; + + public BuildRetentionServiceTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"tiger-retention-test-{Guid.NewGuid()}.db"); + _db = TigerDatabase.Open(_dbPath); + } + + public void Dispose() + { + _db.Dispose(); + if (File.Exists(_dbPath)) + { + File.Delete(_dbPath); + } + } + + [Fact] + public void CleanupExpiredBuilds_DeletesBuildsPastRetentionAcrossOrganizations() + { + var oldFinishTime = DateTime.UtcNow.AddDays(-15).ToString("o"); + var recentFinishTime = DateTime.UtcNow.AddDays(-10).ToString("o"); + _db.WithCommand(cmd => + { + cmd.CommandText = """ + INSERT INTO builds + (organization, project, build_id, build_number, definition_name, definition_id, status, source_branch, finish_time) + VALUES + ('org1', 'proj', 1, 'old-1', 'pipeline', 1, 'completed', 'main', @old), + ('org2', 'proj', 1, 'old-2', 'pipeline', 1, 'completed', 'main', @old), + ('org1', 'proj', 2, 'recent', 'pipeline', 1, 'completed', 'main', @recent), + ('org1', 'proj', 3, 'unknown-time', 'pipeline', 1, 'completed', 'main', NULL) + """; + cmd.Parameters.AddWithValue("@old", oldFinishTime); + cmd.Parameters.AddWithValue("@recent", recentFinishTime); + cmd.ExecuteNonQuery(); + }); + + var service = new BuildRetentionService(new TigerConfig { BackfillDays = 14 }, _db); + var deleted = service.CleanupExpiredBuilds(); + + Assert.Equal(2, deleted); + var remainingBuilds = _db.WithCommand(cmd => + { + cmd.CommandText = "SELECT organization || ':' || build_id FROM builds ORDER BY organization, build_id"; + using var reader = cmd.ExecuteReader(); + var builds = new List(); + while (reader.Read()) + { + builds.Add(reader.GetString(0)); + } + return string.Join("\n", builds); + }); + Assert.Equal(""" + org1:2 + org1:3 + """.ReplaceLineEndings("\n"), remainingBuilds); + } +} diff --git a/src/Tiger.Tests/TigerConfigTests.cs b/src/Tiger.Tests/TigerConfigTests.cs new file mode 100644 index 0000000..d685ae8 --- /dev/null +++ b/src/Tiger.Tests/TigerConfigTests.cs @@ -0,0 +1,16 @@ +using Xunit; + +namespace Tiger.Tests; + +public class TigerConfigTests +{ + [Fact] + public void BackfillDays_DefaultsToFourteen() + { + var configDirectory = Path.Combine(Path.GetTempPath(), $"tiger-config-test-{Guid.NewGuid()}"); + + var config = TigerConfig.Load(configDirectory); + Assert.Equal(14, new TigerConfig().BackfillDays); + Assert.Equal(14, config.BackfillDays); + } +} diff --git a/src/Tiger/BuildRetentionService.cs b/src/Tiger/BuildRetentionService.cs new file mode 100644 index 0000000..5ee558e --- /dev/null +++ b/src/Tiger/BuildRetentionService.cs @@ -0,0 +1,90 @@ +namespace Tiger; + +/// +/// Periodically removes builds and their associated data after the configured retention period. +/// +public sealed class BuildRetentionService : IDisposable +{ + private static readonly TimeSpan s_cleanupInterval = TimeSpan.FromMinutes(15); + + private readonly TigerConfig _config; + private readonly TigerDatabase _db; + private readonly ServiceLog? _log; + private CancellationTokenSource? _cts; + private Task _cleanupTask = Task.CompletedTask; + + public bool IsRunning => !_cleanupTask.IsCompleted; + + public BuildRetentionService(TigerConfig config, TigerDatabase db, ServiceLog? log = null) + { + _config = config; + _db = db; + _log = log; + } + + public void Start() + { + if (IsRunning) + { + return; + } + + _cts = new CancellationTokenSource(); + _cleanupTask = CleanupLoopAsync(_cts.Token); + } + + public async Task StopAsync() + { + if (_cts is not null) + { + await _cts.CancelAsync(); + try + { + await _cleanupTask; + } + catch (OperationCanceledException) + { + } + _cts.Dispose(); + _cts = null; + _cleanupTask = Task.CompletedTask; + } + } + + private async Task CleanupLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + var deletedCount = CleanupExpiredBuilds(); + _log?.Info("Retention", $"Deleted {deletedCount} build(s) older than {_config.BackfillDays} days."); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _log?.Error("Retention", $"Failed to clean up expired builds: {ex.Message}"); + } + + try + { + await Task.Delay(s_cleanupInterval, ct); + } + catch (OperationCanceledException) + { + break; + } + } + } + + internal int CleanupExpiredBuilds() + { + var cutoff = DateTime.UtcNow.AddDays(-_config.BackfillDays); + return _db.DeleteBuildsOlderThan(cutoff); + } + + public void Dispose() + { + _cts?.Cancel(); + _cts?.Dispose(); + } +} diff --git a/src/Tiger/Commands/ConfigEditor.cs b/src/Tiger/Commands/ConfigEditor.cs index 608eb13..1febdb4 100644 --- a/src/Tiger/Commands/ConfigEditor.cs +++ b/src/Tiger/Commands/ConfigEditor.cs @@ -84,7 +84,7 @@ private void RenderConfig() table.AddColumn("Setting"); table.AddColumn("Value"); table.AddRow("Poll interval", $"{_config.PollIntervalSeconds}s"); - table.AddRow("Backfill days", _config.BackfillDays.ToString()); + table.AddRow("Ingestion window", $"{_config.BackfillDays} days"); AnsiConsole.Write(table); AnsiConsole.WriteLine(); diff --git a/src/Tiger/Commands/DashboardCommand.cs b/src/Tiger/Commands/DashboardCommand.cs index d571fd6..21e0a95 100644 --- a/src/Tiger/Commands/DashboardCommand.cs +++ b/src/Tiger/Commands/DashboardCommand.cs @@ -51,6 +51,9 @@ protected override async Task ExecuteAsync(CommandContext context, Cancella var healthAgent = new HealthAgentService(config, db, serviceLog); healthAgent.Start(); + var retention = new BuildRetentionService(config, db, serviceLog); + retention.Start(); + var backfill = new BuildBackfillService(config, db, ingestion, clientFactory, serviceLog); backfill.Start(); @@ -64,6 +67,7 @@ protected override async Task ExecuteAsync(CommandContext context, Cancella await ingestion.StopAsync(); await poller.StopAsync(); await backfill.StopAsync(); + await retention.StopAsync(); await analysisAgent.StopAsync(); } @@ -266,4 +270,3 @@ void OnEntry() } } } - diff --git a/src/Tiger/Commands/PollerCommands.cs b/src/Tiger/Commands/PollerCommands.cs index 73caa04..89926fb 100644 --- a/src/Tiger/Commands/PollerCommands.cs +++ b/src/Tiger/Commands/PollerCommands.cs @@ -24,6 +24,7 @@ protected override async Task ExecuteAsync(CommandContext context, Cancella var ingestion = new BuildIngestionService(db); var clientFactory = new AzdoClientFactory(tigerContext.AzureCredential); var poller = new BuildPoller(config, db, clientFactory); + var retention = new BuildRetentionService(config, db); poller.OnNewBuilds = ingestion.InsertBuildsAsync; @@ -38,6 +39,7 @@ protected override async Task ExecuteAsync(CommandContext context, Cancella AnsiConsole.WriteLine(); poller.Start(); + retention.Start(); try { @@ -49,6 +51,7 @@ protected override async Task ExecuteAsync(CommandContext context, Cancella AnsiConsole.MarkupLine("[yellow]Stopping poller...[/]"); await poller.StopAsync(); + await retention.StopAsync(); AnsiConsole.MarkupLine("[green]Poller stopped.[/]"); return 0; diff --git a/src/Tiger/TigerConfig.cs b/src/Tiger/TigerConfig.cs index c670a6b..b2a4a8b 100644 --- a/src/Tiger/TigerConfig.cs +++ b/src/Tiger/TigerConfig.cs @@ -70,7 +70,7 @@ public sealed class TigerConfig public int PollIntervalSeconds { get; set; } = 300; [JsonPropertyName("backfillDays")] - public int BackfillDays { get; set; } = 7; + public int BackfillDays { get; set; } = 14; [JsonPropertyName("sources")] public List Sources { get; set; } = []; @@ -111,6 +111,11 @@ public static string GetConfigPath(string configDirectory) => private void Normalize() { + if (BackfillDays <= 0) + { + throw new InvalidOperationException("Backfill days must be greater than zero."); + } + foreach (var source in Sources) { source.RepositoryType = AzdoRepositoryTypes.Normalize(source.RepositoryType); @@ -129,6 +134,6 @@ private void Normalize() Repositories = ["dotnet/roslyn"], } ], - BackfillDays = 3, + BackfillDays = 14, }; } diff --git a/src/Tiger/TigerDatabase.cs b/src/Tiger/TigerDatabase.cs index a43776d..54bdeae 100644 --- a/src/Tiger/TigerDatabase.cs +++ b/src/Tiger/TigerDatabase.cs @@ -498,6 +498,35 @@ DELETE FROM builds }); } + public int DeleteBuildsOlderThan(DateTime cutoff) + { + var expiredBuilds = WithCommand(cmd => + { + cmd.CommandText = """ + SELECT organization, build_id + FROM builds + WHERE finish_time IS NOT NULL + AND datetime(finish_time) < datetime(@cutoff) + """; + cmd.Parameters.AddWithValue("@cutoff", cutoff.ToUniversalTime().ToString("o")); + + var builds = new List<(string Organization, int BuildId)>(); + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + builds.Add((reader.GetString(0), reader.GetInt32(1))); + } + return builds; + }); + + foreach (var (organization, buildId) in expiredBuilds) + { + DeleteBuild(organization, buildId); + } + + return expiredBuilds.Count; + } + /// /// Inserts a new build analysis row with status 'pending'. /// diff --git a/src/Tiger/skills/tiger-data/SKILL.md b/src/Tiger/skills/tiger-data/SKILL.md index 20e107a..30f1a9d 100644 --- a/src/Tiger/skills/tiger-data/SKILL.md +++ b/src/Tiger/skills/tiger-data/SKILL.md @@ -279,6 +279,7 @@ ORDER BY b.finish_time DESC; - `pull_requests`, `known_issues`, and `helix_work_items` are keyed by repository or job (not org/project) - Times are stored as ISO 8601 strings in UTC - Only failed test results are stored (not passing tests) +- Build records and their associated test, timeline, Helix, and analysis rows are automatically deleted after the configured `backfillDays` period (14 days by default). Cleanup runs at startup and every 15 minutes. The same setting controls how far back backfill fetches builds. - The `builds.ingestion_status` column shows whether test/timeline/Helix data is available for a build (tests, timeline, and Helix work items are ingested together in a single atomic pass) - Helix work items are fetched inline as part of build ingestion (no separate helix task) - Known issues are refreshed every 15 minutes; closed issues are kept for 7 days before purging