From 18e79e3aceb86590680bceda4c1e40e0aa422f1b Mon Sep 17 00:00:00 2001 From: feruzm Date: Sun, 23 Aug 2026 06:23:51 +0000 Subject: [PATCH 1/3] feat(promoted): apply the moderation mute list to promoted entries Muting an account from the moderation account is how spam and phishing are kept out of the waves feeds, and the indexer applies that list to every waves query it serves. Promoted entries do not go through it: they are served from here. That left a paid placement as the one surface a muted account could still reach an audience through, and the most prominent one in the feed. Read the list straight from chain (condenser_api.get_following ... "ignore", paged) rather than from another service, so it holds even when the indexer is behind, and cache it for 5 minutes since it only changes when a moderator acts. A failed refresh falls back to the last list we saw and re-arms the short TTL with it, so an unreachable node degrades to slightly stale filtering instead of none, and does not put an RPC call on every request for the duration. With no list at all the feed is served unfiltered rather than failing: a moderation filter that cannot load must not take the feed down. Filtered after the promoted cache is read, not before, so a new mute applies on the mute list's own refresh instead of waiting out the 5-minute promoted cache. --- .../EcencyApi.Tests/ModerationMutesTests.cs | 99 ++++++++++ dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs | 9 + .../Infrastructure/ModerationMutes.cs | 176 ++++++++++++++++++ 3 files changed, 284 insertions(+) create mode 100644 dotnet/EcencyApi.Tests/ModerationMutesTests.cs create mode 100644 dotnet/EcencyApi/Infrastructure/ModerationMutes.cs diff --git a/dotnet/EcencyApi.Tests/ModerationMutesTests.cs b/dotnet/EcencyApi.Tests/ModerationMutesTests.cs new file mode 100644 index 00000000..1b729d2b --- /dev/null +++ b/dotnet/EcencyApi.Tests/ModerationMutesTests.cs @@ -0,0 +1,99 @@ +using System.Text.Json.Nodes; +using EcencyApi.Infrastructure; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The moderation mute filter on promoted entries. The list is fetched from +/// chain, but the parts that can silently break a feed are pure: an empty or +/// unreadable list must leave the feed alone, and a match must remove exactly +/// the muted author's entries and nothing else. +/// +public class ModerationMutesTests +{ + private static JsonArray Entries(params string?[] authors) + { + var arr = new JsonArray(); + foreach (var a in authors) + { + var o = new JsonObject { ["permlink"] = "p-" + (a ?? "none") }; + if (a != null) + { + o["author"] = a; + } + arr.Add(o); + } + return arr; + } + + private static string?[] AuthorsOf(JsonArray arr) => + arr.Select(e => e is JsonObject o && o.TryGetPropertyValue("author", out var a) + ? a?.GetValue() + : null).ToArray(); + + [Fact] + public void AnEmptyMuteListLeavesTheFeedUntouched() + { + var entries = Entries("alice", "bob"); + var result = ModerationMutes.FilterMutedAuthors(entries, ModerationMutes.ToSet(Array.Empty())); + Assert.Equal(new[] { "alice", "bob" }, AuthorsOf(result)); + } + + [Fact] + public void MutedAuthorsAreDroppedAndTheRestKeptInOrder() + { + var entries = Entries("alice", "spammer", "bob", "spammer", "carol"); + var result = ModerationMutes.FilterMutedAuthors(entries, ModerationMutes.ToSet(new[] { "spammer" })); + Assert.Equal(new[] { "alice", "bob", "carol" }, AuthorsOf(result)); + } + + [Fact] + public void MatchingIsCaseInsensitive() + { + // Hive account names are lowercase, but nothing here guarantees the two + // sides were normalized by the same code path. + var entries = Entries("Spammer"); + var result = ModerationMutes.FilterMutedAuthors(entries, ModerationMutes.ToSet(new[] { "spammer" })); + Assert.Empty(result); + } + + [Fact] + public void AnEntryWithNoAuthorIsKept() + { + // An unreadable shape is not evidence of anything; dropping it would + // shrink the feed for a reason nobody could see. + var entries = Entries("alice", null); + var result = ModerationMutes.FilterMutedAuthors(entries, ModerationMutes.ToSet(new[] { "spammer" })); + Assert.Equal(2, result.Count); + } + + [Fact] + public void FilteringEveryEntryYieldsAnEmptyArrayNotNull() + { + var entries = Entries("spammer", "spammer"); + var result = ModerationMutes.FilterMutedAuthors(entries, ModerationMutes.ToSet(new[] { "spammer" })); + Assert.NotNull(result); + Assert.Empty(result); + } + + [Fact] + public void ReadFollowingTakesTheFollowingNamesAndSkipsUnusableRows() + { + var rows = new JsonArray( + new JsonObject { ["follower"] = "ecency", ["following"] = "spammer", ["what"] = new JsonArray("ignore") }, + new JsonObject { ["follower"] = "ecency" }, + new JsonObject { ["follower"] = "ecency", ["following"] = "" }, + new JsonObject { ["follower"] = "ecency", ["following"] = "phisher", ["what"] = new JsonArray("ignore") }); + + Assert.Equal(new[] { "spammer", "phisher" }, ModerationMutes.ReadFollowing(rows)); + } + + [Fact] + public void TheModerationAccountIsEcency() + { + // Pinned: this account name is the whole control surface. A typo here + // would read as "nobody is muted" with no error anywhere. + Assert.Equal("ecency", ModerationMutes.Account); + } +} diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs b/dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs index d2fe4f44..2ca30205 100644 --- a/dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs +++ b/dotnet/EcencyApi/Handlers/PrivateApi.Feeds.cs @@ -68,6 +68,15 @@ public static async Task PromotedEntries(HttpContext ctx) var shortContent = double.IsNaN(shortNum) ? 0 : (int)Math.Clamp(shortNum, int.MinValue, int.MaxValue); var posts = await ApiClient.GetPromotedEntries(limit, shortContent); + + // Promoted entries are served from here, not from the waves indexer, so + // the moderation mute list has to be applied on this path too. A muted + // account buying a promoted slot would otherwise land in the most + // prominent position in the feed. Filtered after the cache read rather + // than before it, so a new mute takes effect on the mute list's own + // refresh instead of waiting out the promoted cache. + posts = ModerationMutes.FilterMutedAuthors(posts, await ModerationMutes.Get()); + await ctx.SendJson(200, posts); } diff --git a/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs b/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs new file mode 100644 index 00000000..4e1a8f21 --- /dev/null +++ b/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs @@ -0,0 +1,176 @@ +using System.Text.Json.Nodes; + +namespace EcencyApi.Infrastructure; + +/// +/// Ecency's on-chain moderation mute list. +/// +/// Muting an account from the moderation account is how spam and phishing are +/// kept out of the waves feeds; esync applies that list to every waves query it +/// serves. Promoted entries never go through esync, so without this they were +/// the one surface a muted account could still reach an audience through — and +/// the most prominent one, since a promoted card is a paid placement. +/// +/// Read straight from chain rather than from another service so this holds even +/// if the indexer is behind, and cached because the list changes only when a +/// moderator acts on it. +/// +public static class ModerationMutes +{ + /// The account whose mutes are treated as platform-wide. + public const string Account = "ecency"; + + private const string CacheKey = "moderation-muted-authors"; + + /// + /// Survives a failed refresh, so an unreachable node degrades to the list we + /// last saw rather than to no filtering at all. Never expires on purpose. + /// + private const string LastGoodCacheKey = "moderation-muted-authors-last-good"; + + private const double TtlSeconds = 300; + + /// condenser_api.get_following caps a single response at 1000 rows. + private const int PageSize = 1000; + + /// + /// Bounds the paging loop. 20 pages is 20k muted accounts, far past any real + /// list, so a node that stops advancing the cursor truncates rather than + /// looping forever. + /// + private const int MaxPages = 20; + + /// Replaceable for tests (loopback stub nodes). + internal static HiveRpcClient Rpc = HiveClients.Default; + + /// + /// The muted accounts, cached. Returns an empty set rather than throwing: + /// a moderation filter that cannot load must not take a feed down with it. + /// + public static async Task> Get() + { + var cached = MemCache.Get(CacheKey); + if (cached != null) + { + return ToSet(cached); + } + + try + { + var names = await Fetch(); + MemCache.Set(CacheKey, names, TtlSeconds); + MemCache.Set(LastGoodCacheKey, names); + return ToSet(names); + } + catch (Exception e) + { + Console.WriteLine($"warn: failed to fetch moderation mutes {e.Message}"); + + // Re-arm the short TTL with the stale list so a node outage does not + // put an RPC call on every promoted-entries request for its duration. + var lastGood = MemCache.Get(LastGoodCacheKey); + if (lastGood != null) + { + MemCache.Set(CacheKey, lastGood, TtlSeconds); + return ToSet(lastGood); + } + + return ToSet(Array.Empty()); + } + } + + private static async Task Fetch() + { + var names = new List(); + var start = ""; + + for (var page = 0; page < MaxPages; page++) + { + var result = await Rpc.Call("condenser_api", "get_following", + new JsonArray(Account, start, "ignore", PageSize)); + + if (result is not JsonArray rows || rows.Count == 0) + { + break; + } + + var pageNames = ReadFollowing(rows); + + // `start` is exclusive on Hive, so a page should not repeat the + // cursor. Drop it anyway: against a node treating it as inclusive + // this would re-append the same account until the page cap. + if (pageNames.Count > 0 && pageNames[0] == start) + { + pageNames.RemoveAt(0); + } + + if (pageNames.Count == 0) + { + break; + } + + names.AddRange(pageNames); + + if (rows.Count < PageSize) + { + break; + } + + start = pageNames[^1]; + } + + return names.ToArray(); + } + + internal static List ReadFollowing(JsonArray rows) + { + var names = new List(); + foreach (var row in rows) + { + var name = row is JsonObject o && o.TryGetPropertyValue("following", out var f) + ? f?.GetValue() + : null; + if (!string.IsNullOrEmpty(name)) + { + names.Add(name); + } + } + return names; + } + + internal static HashSet ToSet(IEnumerable names) => + new(names, StringComparer.OrdinalIgnoreCase); + + /// + /// Drop entries authored by a muted account. Returns a new array; entries + /// with no readable author are kept, since an unreadable shape is not + /// evidence of anything and dropping it would silently shrink the feed. + /// + public static JsonArray FilterMutedAuthors(JsonArray entries, ISet muted) + { + if (muted.Count == 0) + { + return entries; + } + + var kept = new JsonArray(); + foreach (var entry in entries.ToArray()) + { + var author = entry is JsonObject o && o.TryGetPropertyValue("author", out var a) + ? a?.GetValue() + : null; + + if (author != null && muted.Contains(author)) + { + continue; + } + + // A node can only live in one parent, and these come from a cache + // clone we own, so detach before re-parenting into the result. + entry?.Parent?.AsArray().Remove(entry); + kept.Add(entry); + } + + return kept; + } +} From 0ae6e1cad30a9fe82347d7e351aa0ad5b7477928 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sun, 23 Aug 2026 08:48:04 +0000 Subject: [PATCH 2/3] review: validate the mute-list response and read its strings leniently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four points from the bot reviews, all fair: Fetch treated any non-array RPC result as "no more rows" and returned what it had, so a node answering 200 with an unusable body cached an empty mute list as both the live and the fallback copy: filtering silently off, nothing anywhere saying so. HiveRpcClient already supports shape validation for exactly this, so pass it and let an unusable answer fail over and, if no node can answer, throw into the existing fallback. An empty list is still served live -- unmuting everyone must take effect -- but no longer overwrites the fallback, so one empty answer cannot turn every later failure into no filtering at all. GetValue() throws on a lone-surrogate escape, which JSON.parse accepts and Hive nodes do emit; that is why JsVal.TryGetStringLenient exists. Reading author and following through it means one malformed name can no longer fail a promoted-entries request or abort a refresh. Drop the Console.WriteLine: this runs on a request path, where the service keeps its logs quiet (CLAUDE.md §6). The fallback, not the log line, is what makes the failure survivable. Serialize refreshes behind a gate. Without one, every request arriving after the TTL lapses started its own paging loop. --- .../EcencyApi.Tests/ModerationMutesTests.cs | 31 +++++++ .../Infrastructure/ModerationMutes.cs | 86 +++++++++++++++---- 2 files changed, 102 insertions(+), 15 deletions(-) diff --git a/dotnet/EcencyApi.Tests/ModerationMutesTests.cs b/dotnet/EcencyApi.Tests/ModerationMutesTests.cs index 1b729d2b..a1797465 100644 --- a/dotnet/EcencyApi.Tests/ModerationMutesTests.cs +++ b/dotnet/EcencyApi.Tests/ModerationMutesTests.cs @@ -89,6 +89,37 @@ public void ReadFollowingTakesTheFollowingNamesAndSkipsUnusableRows() Assert.Equal(new[] { "spammer", "phisher" }, ModerationMutes.ReadFollowing(rows)); } + [Fact] + public void AnAuthorThatCannotBeReadAsAStringDoesNotThrow() + { + // GetValue() throws on a lone-surrogate escape, which JSON.parse + // accepts and Hive nodes do emit. Throwing here would fail the whole + // promoted-entries request over one malformed name. + var entries = new JsonArray( + new JsonObject { ["author"] = 42 }, + new JsonObject { ["author"] = JsonValue.Create((string?)null) }, + new JsonObject { ["author"] = new JsonObject() }, + new JsonObject { ["author"] = "spammer" }); + + var result = ModerationMutes.FilterMutedAuthors(entries, ModerationMutes.ToSet(new[] { "spammer" })); + + // The three unreadable ones survive; only the match is dropped. + Assert.Equal(3, result.Count); + } + + [Fact] + public void AFollowingThatCannotBeReadAsAStringIsSkippedNotFatal() + { + // Same reasoning on the refresh side: one malformed row must not abort + // the mute-list refresh and leave the feed unfiltered. + var rows = new JsonArray( + new JsonObject { ["following"] = 7 }, + new JsonObject { ["following"] = new JsonArray("nested") }, + new JsonObject { ["following"] = "spammer" }); + + Assert.Equal(new[] { "spammer" }, ModerationMutes.ReadFollowing(rows)); + } + [Fact] public void TheModerationAccountIsEcency() { diff --git a/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs b/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs index 4e1a8f21..995201b5 100644 --- a/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs +++ b/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs @@ -43,6 +43,14 @@ public static class ModerationMutes /// Replaceable for tests (loopback stub nodes). internal static HiveRpcClient Rpc = HiveClients.Default; + /// + /// One refresh at a time. Without this, every request arriving after the TTL + /// lapses starts its own paging loop, so a burst turns one refresh into as + /// many RPC conversations as there are concurrent promoted-entries requests. + /// The waiters re-read the cache and take the winner's result. + /// + private static readonly SemaphoreSlim RefreshGate = new(1, 1); + /// /// The muted accounts, cached. Returns an empty set rather than throwing: /// a moderation filter that cannot load must not take a feed down with it. @@ -55,27 +63,58 @@ public static async Task> Get() return ToSet(cached); } + await RefreshGate.WaitAsync(); + try + { + // Someone else may have refreshed while this request queued. + cached = MemCache.Get(CacheKey); + if (cached != null) + { + return ToSet(cached); + } + + return ToSet(await Refresh()); + } + finally + { + RefreshGate.Release(); + } + } + + private static async Task Refresh() + { try { var names = await Fetch(); MemCache.Set(CacheKey, names, TtlSeconds); - MemCache.Set(LastGoodCacheKey, names); - return ToSet(names); + + // Only a list with something in it is worth falling back to. An empty + // one is served live (unmuting everyone must take effect) but must not + // overwrite the fallback, or one empty answer would turn every later + // failure into no filtering at all. + if (names.Length > 0) + { + MemCache.Set(LastGoodCacheKey, names); + } + + return names; } - catch (Exception e) + catch (Exception) { - Console.WriteLine($"warn: failed to fetch moderation mutes {e.Message}"); - + // Deliberately silent: this runs on the promoted-entries request path + // and the service keeps its logs quiet there (CLAUDE.md, "No hot-path + // logging"). The fallback below is what makes the failure survivable. + // // Re-arm the short TTL with the stale list so a node outage does not // put an RPC call on every promoted-entries request for its duration. var lastGood = MemCache.Get(LastGoodCacheKey); if (lastGood != null) { MemCache.Set(CacheKey, lastGood, TtlSeconds); - return ToSet(lastGood); + return lastGood; } - return ToSet(Array.Empty()); + return Array.Empty(); } } @@ -86,10 +125,17 @@ private static async Task Fetch() for (var page = 0; page < MaxPages; page++) { + // Validate the shape at the client, so a node answering 200 with + // something that is not a row array fails over to another one and, + // if none can answer, throws. Without this an unusable answer read + // as "no more rows" and the caller cached an empty mute list -- + // filtering silently off, with nothing anywhere saying so. var result = await Rpc.Call("condenser_api", "get_following", - new JsonArray(Account, start, "ignore", PageSize)); + new JsonArray(Account, start, "ignore", PageSize), + validateResult: r => r is JsonArray); - if (result is not JsonArray rows || rows.Count == 0) + var rows = (JsonArray)result!; + if (rows.Count == 0) { break; } @@ -122,14 +168,26 @@ private static async Task Fetch() return names.ToArray(); } + /// + /// Read one string property the lenient way. `GetValue<string>()` throws on + /// a lone-surrogate escape, which JSON.parse accepts and Hive nodes do emit; + /// letting that throw here would fail a promoted-entries request, or abort a + /// mute-list refresh, over one malformed account name. + /// + private static string? ReadString(JsonNode? owner, string property) => + owner is JsonObject o + && o.TryGetPropertyValue(property, out var node) + && node is JsonValue value + && JsVal.TryGetStringLenient(value, out var s) + ? s + : null; + internal static List ReadFollowing(JsonArray rows) { var names = new List(); foreach (var row in rows) { - var name = row is JsonObject o && o.TryGetPropertyValue("following", out var f) - ? f?.GetValue() - : null; + var name = ReadString(row, "following"); if (!string.IsNullOrEmpty(name)) { names.Add(name); @@ -156,9 +214,7 @@ public static JsonArray FilterMutedAuthors(JsonArray entries, ISet muted var kept = new JsonArray(); foreach (var entry in entries.ToArray()) { - var author = entry is JsonObject o && o.TryGetPropertyValue("author", out var a) - ? a?.GetValue() - : null; + var author = ReadString(entry, "author"); if (author != null && muted.Contains(author)) { From 834ee382357b82bfd14c3b23bc6c4444366e2cff Mon Sep 17 00:00:00 2001 From: feruzm Date: Sun, 23 Aug 2026 09:21:58 +0000 Subject: [PATCH 3/3] fix: cache a failed mute refresh, and do not queue behind a slow one Found reviewing my own fix for the refresh stampede: the gate made the outage case worse than having no gate at all. On failure nothing was cached, so every request that queued behind the refresh woke up, re-read an empty cache and ran its own full node-failover sweep. The Nth caller paid N times the timeout budget, and the queue grew as long as the pool stayed down. Cache the failure for 30s, the empty result included. One retry goes on the clock instead of one per request, and a blip costs a single interval of stale filtering. Also bound the wait on the gate: a refresh is normally one RPC round trip, so waiters still get the real list, but a pool that is timing out no longer hands its latency to a promoted-entries request. The test counts RPC attempts rather than elapsed time. The first version measured duration and passed with the fix removed -- a refused connection fails in microseconds, so timing cannot tell the two apart. Counting attempts gives 1 with the fix and 9 without. --- .../EcencyApi.Tests/ModerationMutesTests.cs | 69 +++++++++++++++++++ .../Infrastructure/ModerationMutes.cs | 48 +++++++++---- 2 files changed, 104 insertions(+), 13 deletions(-) diff --git a/dotnet/EcencyApi.Tests/ModerationMutesTests.cs b/dotnet/EcencyApi.Tests/ModerationMutesTests.cs index a1797465..7b84c107 100644 --- a/dotnet/EcencyApi.Tests/ModerationMutesTests.cs +++ b/dotnet/EcencyApi.Tests/ModerationMutesTests.cs @@ -12,6 +12,9 @@ namespace EcencyApi.Tests; /// public class ModerationMutesTests { + private static long TotalRpcCalls() => + ModerationMutes.Rpc.HealthSnapshot().Sum(n => n!["calls"]!.GetValue()); + private static JsonArray Entries(params string?[] authors) { var arr = new JsonArray(); @@ -120,6 +123,72 @@ public void AFollowingThatCannotBeReadAsAStringIsSkippedNotFatal() Assert.Equal(new[] { "spammer" }, ModerationMutes.ReadFollowing(rows)); } + [Fact] + public async Task AFailedRefreshIsCachedSoQueuedRequestsDoNotEachRetry() + { + // The refresh gate serializes callers. Without caching the failure, a + // dead node pool makes that worse than no gate at all: each queued + // request waits out the one ahead of it and then runs its own full + // failover sweep, so the Nth caller pays N times the timeout budget. + var original = ModerationMutes.Rpc; + MemCache.Del("moderation-muted-authors"); + MemCache.Del("moderation-muted-authors-last-good"); + + // Port 9 (discard) refuses immediately, so this measures the code path + // rather than a real network timeout. + ModerationMutes.Rpc = new HiveRpcClient( + new[] { "http://127.0.0.1:9/" }, timeoutMs: 250, failoverThreshold: 1); + + try + { + var first = await ModerationMutes.Get(); + Assert.Empty(first); + + // Count attempts rather than elapsed time: a refused connection + // fails in microseconds, so a timing assertion passes just as + // happily whether or not the failure was cached. + var callsAfterFirst = TotalRpcCalls(); + + var followers = await Task.WhenAll( + Enumerable.Range(0, 8).Select(_ => ModerationMutes.Get())); + + Assert.All(followers, f => Assert.Empty(f)); + Assert.Equal(callsAfterFirst, TotalRpcCalls()); + } + finally + { + ModerationMutes.Rpc = original; + MemCache.Del("moderation-muted-authors"); + MemCache.Del("moderation-muted-authors-last-good"); + } + } + + [Fact] + public async Task AFailedRefreshFallsBackToTheLastListSeen() + { + var original = ModerationMutes.Rpc; + MemCache.Del("moderation-muted-authors"); + + // Stand in for a previously successful fetch. + MemCache.Set("moderation-muted-authors-last-good", new[] { "spammer" }); + ModerationMutes.Rpc = new HiveRpcClient( + new[] { "http://127.0.0.1:9/" }, timeoutMs: 250, failoverThreshold: 1); + + try + { + // Stale filtering, not no filtering: an unreachable pool must not be + // a way for a muted account back into the feed. + var muted = await ModerationMutes.Get(); + Assert.Equal(new[] { "spammer" }, muted.OrderBy(x => x)); + } + finally + { + ModerationMutes.Rpc = original; + MemCache.Del("moderation-muted-authors"); + MemCache.Del("moderation-muted-authors-last-good"); + } + } + [Fact] public void TheModerationAccountIsEcency() { diff --git a/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs b/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs index 995201b5..2166651d 100644 --- a/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs +++ b/dotnet/EcencyApi/Infrastructure/ModerationMutes.cs @@ -30,6 +30,22 @@ public static class ModerationMutes private const double TtlSeconds = 300; + /// + /// How long a failed refresh is held before trying again. Short, so a blip + /// costs one interval of stale filtering, but not zero: caching the failure + /// is what stops every request behind the gate from running its own full + /// node-failover sweep. + /// + private const double FailureTtlSeconds = 30; + + /// + /// How long a request waits for someone else's refresh before answering from + /// what it already has. A normal refresh is one RPC round trip, so waiters + /// get the real list; this only bounds the pathological case where the whole + /// node pool is timing out and the refresh takes tens of seconds. + /// + private static readonly TimeSpan RefreshWait = TimeSpan.FromSeconds(2); + /// condenser_api.get_following caps a single response at 1000 rows. private const int PageSize = 1000; @@ -63,7 +79,14 @@ public static async Task> Get() return ToSet(cached); } - await RefreshGate.WaitAsync(); + if (!await RefreshGate.WaitAsync(RefreshWait)) + { + // A refresh is already running and is taking far longer than one RPC + // round trip. Queueing behind it would hand that latency to a + // promoted-entries request, so answer from the fallback instead. + return ToSet(MemCache.Get(LastGoodCacheKey) ?? Array.Empty()); + } + try { // Someone else may have refreshed while this request queued. @@ -103,18 +126,17 @@ private static async Task Refresh() { // Deliberately silent: this runs on the promoted-entries request path // and the service keeps its logs quiet there (CLAUDE.md, "No hot-path - // logging"). The fallback below is what makes the failure survivable. - // - // Re-arm the short TTL with the stale list so a node outage does not - // put an RPC call on every promoted-entries request for its duration. - var lastGood = MemCache.Get(LastGoodCacheKey); - if (lastGood != null) - { - MemCache.Set(CacheKey, lastGood, TtlSeconds); - return lastGood; - } - - return Array.Empty(); + // logging"). The caching below is what makes the failure survivable. + var fallback = MemCache.Get(LastGoodCacheKey) ?? Array.Empty(); + + // Cache the failure, including the empty one. Without this the gate + // above turns a node outage into something worse than no gate at all: + // each queued request waits out the one ahead of it and then runs its + // own full failover sweep, so the Nth caller pays N times the timeout + // budget. Caching lets every waiter answer immediately and puts one + // retry on the clock instead of one per request. + MemCache.Set(CacheKey, fallback, FailureTtlSeconds); + return fallback; } }