From b2f9617c09e59a634a169535c8346ba0abea29a9 Mon Sep 17 00:00:00 2001 From: feruzm Date: Tue, 11 Aug 2026 11:39:00 +0000 Subject: [PATCH 1/2] fix(portfolio): keep engine balances when an enrichment leg fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FetchEngineTokensWithBalance awaited token metadata and market metrics alongside the balances it had already fetched. Either one throwing after its failover was exhausted hit the outer catch, which returns an empty array — so a metrics outage surfaced identically to a total Hive-Engine outage: no tokens in the wallet at all. Only balances are load-bearing. Tokens supply name/precision/icon and metrics supply the price used for fiat valuation; ConvertEngineToken already null-tolerates both. Degrade them to empty, as the rewards leg already does, so the balance rows still render (unpriced) instead of the layer disappearing. --- .../EcencyApi.Tests/EngineOptionalLegTests.cs | 56 +++++++++++++++++++ dotnet/EcencyApi/Handlers/WalletApi.Engine.cs | 26 ++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 dotnet/EcencyApi.Tests/EngineOptionalLegTests.cs diff --git a/dotnet/EcencyApi.Tests/EngineOptionalLegTests.cs b/dotnet/EcencyApi.Tests/EngineOptionalLegTests.cs new file mode 100644 index 00000000..0c93a17b --- /dev/null +++ b/dotnet/EcencyApi.Tests/EngineOptionalLegTests.cs @@ -0,0 +1,56 @@ +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using EcencyApi.Infrastructure; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The engine layer is assembled from one required leg (balances) and three +/// enrichment legs (token metadata, market metrics, unclaimed rewards). Losing +/// enrichment must cost the decoration, not the balances: a metrics outage used +/// to surface identically to a total Hive-Engine outage — no tokens at all. +/// +public class EngineOptionalLegTests +{ + [Fact] + public async Task Optional_PassesThroughASuccessfulLeg() + { + var expected = new JsonArray { new JsonObject { ["symbol"] = "LEO" } }; + + var result = await WalletApi.Optional(Task.FromResult(expected), "engine tokens"); + + Assert.Single(result); + Assert.Equal("LEO", result[0]!["symbol"]!.GetValue()); + } + + [Fact] + public async Task Optional_DegradesAFailedLegToEmptyInsteadOfThrowing() + { + var failed = Task.FromException(new Exception("upstream down")); + + var result = await WalletApi.Optional(failed, "engine metrics"); + + Assert.Empty(result); + } + + // Enrichment legs must not be able to take the layer down between them: even + // with both token metadata and metrics failing, the balances still render + // (unpriced) rather than the wallet showing no engine tokens at all. + [Fact] + public async Task Optional_LetsBalancesSurviveLosingEveryEnrichmentLeg() + { + var tokens = WalletApi.Optional( + Task.FromException(new Exception("tokens down")), "engine tokens"); + var metrics = WalletApi.Optional( + Task.FromException(new Exception("metrics down")), "engine metrics"); + + Assert.Empty(await tokens); + Assert.Empty(await metrics); + + // ConvertEngineToken null-tolerates both, so a balance row still converts. + var converted = EcencyApi.Models.HiveEngine.ConvertEngineToken( + new JsonObject { ["symbol"] = "LEO", ["balance"] = "1.5" }, null, null, null); + Assert.Equal("LEO", JsVal.AsString(JsVal.Prop(converted, "symbol"))); + } +} diff --git a/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs b/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs index 595b0860..75911218 100644 --- a/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs +++ b/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs @@ -154,6 +154,21 @@ public static async Task FetchEngineRewards(string username) } } + /// + /// Degrades an enrichment leg to an empty result instead of failing the caller. + /// The engine layer must survive losing decoration; only the balances leg is + /// load-bearing. Logged once so a persistent upstream outage is still visible. + /// + internal static async Task Optional(Task task, string label) + { + try { return await task; } + catch (Exception err) + { + Console.WriteLine($"failed to get {label} {err.Message}"); + return new JsonArray(); + } + } + private static async Task FetchEngineTokensWithBalance(string username) { try @@ -163,8 +178,15 @@ private static async Task FetchEngineTokensWithBalance(string usernam var symbols = balances.Select(b => JsVal.AsString(JsVal.Prop(b, "symbol")) ?? JsVal.ToJsString(JsVal.Prop(b, "symbol"))) .Where(s => s != null).Select(s => s!).ToList(); - var tokensTask = FetchEngineTokens(symbols); - var metricsTask = FetchEngineMetrics(symbols); + // Balances are the only required leg: they are the user's actual + // holdings. Tokens (name/precision/icon) and metrics (market price, + // used for fiat valuation) are enrichment — ConvertEngineToken already + // null-tolerates both. Letting either failure propagate would hit the + // catch below and blank the whole layer, so a metrics outage would look + // identical to a total Hive-Engine outage: no tokens in the wallet at + // all. Degrade to unpriced balances instead of showing nothing. + var tokensTask = Optional(FetchEngineTokens(symbols), "engine tokens"); + var metricsTask = Optional(FetchEngineMetrics(symbols), "engine metrics"); // The rewards upstream allows 30s, but this whole fetch must fit // the portfolioV2 engine leg budget (4.5s) — a slow rewards call // would otherwise blank the entire engine layer. Rewards are From 8f9ea7d79cb639e53bcd7fd317887264c8220caa Mon Sep 17 00:00:00 2001 From: feruzm Date: Tue, 11 Aug 2026 11:46:28 +0000 Subject: [PATCH 2/2] review: bound the optional legs, don't only catch them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Catching exceptions alone left the stalling case unhandled, which is the outage that actually matters: EngineRpcClient.Find walks the whole node pool at 2s per attempt, far outlasting the engine leg's budget. Task.WhenAll stayed pending until PortfolioV2's own timeout fired and returned an empty layer — discarding the balances this change exists to preserve. Optional now bounds each leg as well as catching it, at the same budget the rewards leg already uses. The inner wrapper never faults, so timing a leg out cannot leave an unobserved exception behind. --- .../EcencyApi.Tests/EngineOptionalLegTests.cs | 30 ++++++++++++++++--- dotnet/EcencyApi/Handlers/WalletApi.Engine.cs | 22 +++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/dotnet/EcencyApi.Tests/EngineOptionalLegTests.cs b/dotnet/EcencyApi.Tests/EngineOptionalLegTests.cs index 0c93a17b..68acb7a7 100644 --- a/dotnet/EcencyApi.Tests/EngineOptionalLegTests.cs +++ b/dotnet/EcencyApi.Tests/EngineOptionalLegTests.cs @@ -18,7 +18,7 @@ public async Task Optional_PassesThroughASuccessfulLeg() { var expected = new JsonArray { new JsonObject { ["symbol"] = "LEO" } }; - var result = await WalletApi.Optional(Task.FromResult(expected), "engine tokens"); + var result = await WalletApi.Optional(Task.FromResult(expected), "engine tokens", 2000); Assert.Single(result); Assert.Equal("LEO", result[0]!["symbol"]!.GetValue()); @@ -29,11 +29,33 @@ public async Task Optional_DegradesAFailedLegToEmptyInsteadOfThrowing() { var failed = Task.FromException(new Exception("upstream down")); - var result = await WalletApi.Optional(failed, "engine metrics"); + var result = await WalletApi.Optional(failed, "engine metrics", 2000); Assert.Empty(result); } + // A stalling upstream, not a throwing one, is the outage that matters here: + // the engine node pool is walked at 2s per attempt and far outlasts the leg + // budget, so catching exceptions alone would leave the caller waiting until + // its own timeout returned an empty layer — losing the balances entirely. + [Fact] + public async Task Optional_BoundsAStallingLeg() + { + var stalled = new TaskCompletionSource(); + + var started = System.Diagnostics.Stopwatch.StartNew(); + var result = await WalletApi.Optional(stalled.Task, "engine metrics", 150); + started.Stop(); + + Assert.Empty(result); + Assert.True(started.ElapsedMilliseconds < 2000, + $"should have given up near the timeout, took {started.ElapsedMilliseconds}ms"); + + // Completing late must not fault anything the caller already moved past. + stalled.SetException(new Exception("late failure")); + await Task.Delay(50); + } + // Enrichment legs must not be able to take the layer down between them: even // with both token metadata and metrics failing, the balances still render // (unpriced) rather than the wallet showing no engine tokens at all. @@ -41,9 +63,9 @@ public async Task Optional_DegradesAFailedLegToEmptyInsteadOfThrowing() public async Task Optional_LetsBalancesSurviveLosingEveryEnrichmentLeg() { var tokens = WalletApi.Optional( - Task.FromException(new Exception("tokens down")), "engine tokens"); + Task.FromException(new Exception("tokens down")), "engine tokens", 2000); var metrics = WalletApi.Optional( - Task.FromException(new Exception("metrics down")), "engine metrics"); + Task.FromException(new Exception("metrics down")), "engine metrics", 2000); Assert.Empty(await tokens); Assert.Empty(await metrics); diff --git a/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs b/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs index 75911218..1efbfb92 100644 --- a/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs +++ b/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs @@ -157,9 +157,20 @@ public static async Task FetchEngineRewards(string username) /// /// Degrades an enrichment leg to an empty result instead of failing the caller. /// The engine layer must survive losing decoration; only the balances leg is - /// load-bearing. Logged once so a persistent upstream outage is still visible. + /// load-bearing. + /// + /// Bounded as well as caught: a stalling upstream is the more common outage, + /// and EngineRpcClient.Find walks the whole node pool at 2s per attempt, which + /// far outlasts the engine leg's budget. Catching exceptions alone would leave + /// Task.WhenAll pending until the caller's own timeout fired and returned an + /// empty layer — discarding the balances this exists to preserve. /// - internal static async Task Optional(Task task, string label) + internal static Task Optional(Task task, string label, int timeoutMs) => + WithTimeout(Observed(task, label), timeoutMs, new JsonArray()); + + /// Never faults, so timing it out cannot leave an unobserved exception behind. + /// Logged once so a persistent upstream outage is still visible. + private static async Task Observed(Task task, string label) { try { return await task; } catch (Exception err) @@ -185,8 +196,8 @@ private static async Task FetchEngineTokensWithBalance(string usernam // catch below and blank the whole layer, so a metrics outage would look // identical to a total Hive-Engine outage: no tokens in the wallet at // all. Degrade to unpriced balances instead of showing nothing. - var tokensTask = Optional(FetchEngineTokens(symbols), "engine tokens"); - var metricsTask = Optional(FetchEngineMetrics(symbols), "engine metrics"); + var tokensTask = Optional(FetchEngineTokens(symbols), "engine tokens", EngineEnrichmentTimeoutMs); + var metricsTask = Optional(FetchEngineMetrics(symbols), "engine metrics", EngineEnrichmentTimeoutMs); // The rewards upstream allows 30s, but this whole fetch must fit // the portfolioV2 engine leg budget (4.5s) — a slow rewards call // would otherwise blank the entire engine layer. Rewards are @@ -322,6 +333,9 @@ await Task.WhenAll(globalPropsTask, accountTask, marketTask, pointsTask, engineT private const int FastLegTimeout = 3000; private const int SlowLegTimeout = 4500; private const int EngineRewardsTimeoutMs = 2000; + // Same budget as rewards: balances must complete first, then the enrichment + // legs run concurrently, and the whole engine fetch has to fit SlowLegTimeout. + private const int EngineEnrichmentTimeoutMs = 2000; private static async Task WithTimeout(Task task, int ms, T fallback) {