fix(portfolio): route account fetches around metadata-stripping RPC nodes

dotnet/EcencyApi.Tests/HiveRpcFailoverTests.cs

@@ -21,6 +21,16 @@ private sealed class StubNode : IAsyncDisposable
         public string Url { get; }
         public int Hits;
 
+        /// <summary>Whether this node serves account metadata. Nodes that strip it
+        /// answer with a well-formed account whose posting_json_metadata is empty.</summary>
+        public bool ServesMetadata = true;
+
+        // A healthy node's get_accounts result. Account-metadata presence matters:
+        // GetAccounts prefers a node that serves it, so the default stub carries it.
+        private string AccountResultBody =>
+            "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[{\"name\":\"served-by\",\"port\":\"" + Url
+            + "\",\"posting_json_metadata\":" + (ServesMetadata ? "\"{\\\"profile\\\":{}}\"" : "\"\"") + "}]}";
+
         public StubNode(Func<int> handler)
         {
             _handler = handler;
@@ -45,7 +55,7 @@ private async Task Loop()
                 if (status == 200)
                 {
                     body = Encoding.UTF8.GetBytes(
-                        "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[{\"name\":\"served-by\",\"port\":\"" + Url + "\"}]}");
+                        AccountResultBody);
                 }
                 else if (status == -1)
                 {
@@ -60,7 +70,7 @@ private async Task Loop()
                     // 1s unproven prior, below any test timeout).
                     await Task.Delay(1500);
                     body = Encoding.UTF8.GetBytes(
-                        "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[{\"name\":\"served-by\",\"port\":\"" + Url + "\"}]}");
+                        AccountResultBody);
                     status = 200;
                 }
                 else if (status == -3)
@@ -70,6 +80,22 @@ private async Task Loop()
                     body = Encoding.UTF8.GetBytes("{\"jsonrpc\":\"2.0\",\"id\":1}");
                     status = 200;
                 }
+                else if (status == -4)
+                {
+                    // RPC-level error: the node answered, the error is the
+                    // application's (no failover, no health penalty).
+                    body = Encoding.UTF8.GetBytes(
+                        "{\"jsonrpc\":\"2.0\",\"id\":1,\"error\":{\"message\":\"boom\"}}");
+                    status = 200;
+                }
+                else if (status == -5)
+                {
+                    // Well-formed array whose entries are scalars: passes a bare
+                    // "is an array" check but carries no readable account.
+                    body = Encoding.UTF8.GetBytes(
+                        "{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":[\"invalid\"]}");
+                    status = 200;
+                }
                 else
                 {
                     body = Encoding.UTF8.GetBytes("rate limited");
@@ -211,6 +237,99 @@ public async Task MalformedResultNode_IsDemotedOnSubsequentCalls()
         Assert.True(good.Hits >= 2);
     }
 
+    // A node can strip account metadata: balances and reputation are correct but
+    // posting_json_metadata comes back empty. That is a well-formed array, so shape
+    // validation passes and the latency EWMA happily keeps such a node first —
+    // silently blanking portfolio engine/chain token visibility, which is derived
+    // entirely from that field. GetAccounts routes around it.
+    [Fact]
+    public async Task MetadataStrippingNode_IsSkippedForAccountFetches()
+    {
+        await using var stripped = new StubNode(() => 200) { ServesMetadata = false };
+        await using var full = new StubNode(() => 200);
+
+        var client = new HiveRpcClient(new[] { stripped.Url, full.Url }, timeoutMs: 1500);
+
+        var accounts = await client.GetAccounts(new[] { "good-karma" });
+
+        Assert.NotNull(accounts);
+        var meta = accounts![0]!["posting_json_metadata"]!.GetValue<string>();
+        Assert.False(string.IsNullOrEmpty(meta), "should have used the node serving metadata");
+        Assert.Equal(1, stripped.Hits); // consulted once, no same-node retry
+        Assert.True(full.Hits >= 1);
+    }
+
+    // The preference is soft: an account that genuinely has no metadata looks
+    // identical to a stripped response, so once no node can do better the answer
+    // is returned rather than failing the request.
+    [Fact]
+    public async Task NoNodeServesMetadata_StillReturnsTheAccount()
+    {
+        await using var a = new StubNode(() => 200) { ServesMetadata = false };
+        await using var b = new StubNode(() => 200) { ServesMetadata = false };
+
+        var client = new HiveRpcClient(new[] { a.Url, b.Url }, timeoutMs: 1500);
+
+        var accounts = await client.GetAccounts(new[] { "good-karma" });
+
+        Assert.NotNull(accounts);
+        Assert.Equal("served-by", accounts![0]!["name"]!.GetValue<string>());
+        // The *first* well-formed answer is the floor, not whichever probe ran last.
+        Assert.Equal(a.Url, accounts[0]!["port"]!.GetValue<string>());
+    }
+
+    // The probe is optional, so its failure must not fail the request: an RPC-level
+    // error from the node we only consulted to improve on an answer we already hold
+    // would otherwise turn a call that previously succeeded into a hard failure.
+    [Fact]
+    public async Task PreferenceProbeHittingRpcError_StillReturnsTheFirstAnswer()
+    {
+        await using var stripped = new StubNode(() => 200) { ServesMetadata = false };
+        await using var erroring = new StubNode(() => -4);
+
+        var client = new HiveRpcClient(new[] { stripped.Url, erroring.Url }, timeoutMs: 1500);
+
+        var accounts = await client.GetAccounts(new[] { "good-karma" });
+
+        Assert.NotNull(accounts);
+        Assert.Equal(stripped.Url, accounts![0]!["port"]!.GetValue<string>());
+    }
+
+    // A node answering with scalar entries passes a bare "is an array" check but
+    // reads as empty downstream — the same silent blanking a metadata-stripping
+    // node causes, so it must fail over rather than be accepted.
+    [Fact]
+    public async Task ScalarAccountArray_FailsOverAsUnusable()
+    {
+        await using var scalar = new StubNode(() => -5);
+        await using var good = new StubNode(() => 200);
+
+        var client = new HiveRpcClient(new[] { scalar.Url, good.Url }, timeoutMs: 1500);
+
+        var accounts = await client.GetAccounts(new[] { "good-karma" });
+
+        Assert.NotNull(accounts);
+        Assert.Equal(good.Url, accounts![0]!["port"]!.GetValue<string>());
+        Assert.Equal(1, scalar.Hits); // unusable result advances immediately
+    }
+
+    // Probing is bounded: an account with no metadata is a common case that no node
+    // can satisfy, so the pool must not be swept on every such request.
+    [Fact]
+    public async Task MetadataPreference_ProbesAtMostTwoNodes()
+    {
+        await using var a = new StubNode(() => 200) { ServesMetadata = false };
+        await using var b = new StubNode(() => 200) { ServesMetadata = false };
+        await using var c = new StubNode(() => 200) { ServesMetadata = false };
+        await using var d = new StubNode(() => 200) { ServesMetadata = false };
+
+        var client = new HiveRpcClient(new[] { a.Url, b.Url, c.Url, d.Url }, timeoutMs: 1500);
+
+        Assert.NotNull(await client.GetAccounts(new[] { "good-karma" }));
+
+        Assert.Equal(2, a.Hits + b.Hits + c.Hits + d.Hits);
+    }
+
     [Fact]
     public async Task AllNodesMalformed_ThrowsNamingTheNode()
     {

dotnet/EcencyApi/Infrastructure/HiveRpcClient.cs

@@ -42,6 +42,10 @@ public RpcException(string message) : base(message) { }
 
     private static long NowMs => Environment.TickCount64;
 
+    /// <summary>How many nodes may be consulted to satisfy a soft result
+    /// preference before the first well-formed answer is accepted as-is.</summary>
+    private const int MaxPreferenceProbes = 2;
+
     // ---- calls -------------------------------------------------------------
 
     /// <param name="validateResult">Optional shape check for the RPC result. A node
@@ -50,8 +54,21 @@ public RpcException(string message) : base(message) { }
     /// yielding no array); without validation that response counts as a SUCCESS,
     /// so the health tracker keeps the poisoned node ranked first for the whole
     /// window. A failed check is treated as node failure and fails over.</param>
+    /// <param name="preferResult">Optional *soft* check: the result is well-formed
+    /// but this node cannot serve the caller's needs. Unlike validateResult this is
+    /// not a health signal — the node is fine for other calls — so it is neither
+    /// retried nor marked unhealthy; we simply move on and keep its answer. If no
+    /// node satisfies the preference, the first such answer is returned rather than
+    /// throwing, so the caller is never worse off than without the preference.
+    ///
+    /// Exists because some Hive nodes serve accounts with account metadata stripped:
+    /// balances and reputation are correct, posting_json_metadata is empty. That is a
+    /// valid 200 with a usable array, so shape validation passes and the latency EWMA
+    /// keeps such a node ranked first — silently blanking every metadata-derived
+    /// feature (portfolio engine/chain token visibility) with no error and no log.</param>
     public async Task<JsonNode?> Call(string api, string method, JsonNode @params,
-        Func<JsonNode?, bool>? validateResult = null)
+        Func<JsonNode?, bool>? validateResult = null,
+        Func<JsonNode?, bool>? preferResult = null)
     {
         var request = new JsonObject
         {
@@ -65,6 +82,9 @@ public RpcException(string message) : base(message) { }
         var body = JsJson.Stringify(request);
 
         Exception? lastError = null;
+        JsonNode? unpreferred = null;
+        var haveUnpreferred = false;
+        var unpreferredCount = 0;
 
         foreach (var nodeIndex in _health.OrderedNodeIndices())
         {
@@ -84,14 +104,35 @@ public RpcException(string message) : base(message) { }
                             $"RPC node {node} returned unusable {method} result",
                             advanceImmediately: true);
                     }
+                    // The node is healthy either way — record the success before
+                    // deciding whether its answer is the one we wanted.
                     _health.RecordSuccess(nodeIndex, NowMs - started);
+                    if (preferResult != null && !preferResult(result))
+                    {
+                        // Keep the first such answer as the floor and try the next
+                        // node; same-node retry would return the same thing.
+                        if (!haveUnpreferred) { unpreferred = result; haveUnpreferred = true; }
+                        // Bounded on purpose. Roughly an eighth of active accounts
+                        // genuinely carry no metadata, and for those NO node can
+                        // satisfy the preference — probing the whole pool every time
+                        // would multiply RPC load on a common case to route around a
+                        // rare one. One alternative is enough to get past a single
+                        // metadata-stripping node, which is all this guards against.
+                        if (++unpreferredCount >= MaxPreferenceProbes) return unpreferred;
+                        break;
+                    }
                     return result;
                 }
                 catch (RpcException)
                 {
                     // The node answered; the error is the application's. No
                     // failover (dhive semantics), and no failure mark.
                     _health.RecordSuccess(nodeIndex, NowMs - started);
+                    // ...but if we only came to this node to improve on an answer we
+                    // already hold, its error belongs to the optional probe, not to
+                    // the caller's request. Rethrowing here would fail a call that
+                    // would have succeeded without the preference.
+                    if (haveUnpreferred) return unpreferred;
                     throw;
                 }
                 catch (NodeUnavailableException e)
@@ -118,6 +159,11 @@ public RpcException(string message) : base(message) { }
             }
         }
 
+        // No node satisfied the preference, but one answered well-formed: that is
+        // the normal outcome when the preference is genuinely unsatisfiable (an
+        // account really has no metadata), so return it instead of failing.
+        if (haveUnpreferred) return unpreferred;
+
         // Every node exhausted — surface the last transport error (dhive throws
         // after cycling the whole list).
         throw lastError ?? new InvalidOperationException("no RPC nodes configured");
@@ -220,10 +266,56 @@ public NodeUnavailableException(string message, bool advanceImmediately,
             nameArr.Add(n is null ? null : JsonValue.Create(n));
         }
         var result = await Call("condenser_api", "get_accounts", new JsonArray(nameArr),
-            r => r is JsonArray);
+            IsAccountArray,
+            HasAnyAccountMetadata);
         return result as JsonArray;
     }
 
+    /// <summary>
+    /// A usable get_accounts result: an array whose entries are account objects or
+    /// JSON null (an unknown account). A node answering with scalar entries passes a
+    /// bare "is an array" check but yields nothing readable downstream — metadata
+    /// reads off it come back empty, which silently blanks portfolio token
+    /// visibility exactly like a metadata-stripping node. Treat it as node failure.
+    /// </summary>
+    internal static bool IsAccountArray(JsonNode? result)
+    {
+        if (result is not JsonArray accounts) return false;
+
+        foreach (var account in accounts)
+        {
+            if (account is not null and not JsonObject) return false;
+        }
+
+        return true;
+    }
+
+    /// <summary>
+    /// True when at least one returned account carries a non-empty
+    /// posting_json_metadata. Nodes that strip account metadata answer with a
+    /// well-formed array whose entries have it blank; portfolio token visibility
+    /// is derived from that field, so such an answer silently reads as "this user
+    /// enabled nothing". Soft preference, not a health signal: an account that
+    /// genuinely has no metadata produces the same shape, and after every node
+    /// declines the caller still gets the response.
+    /// </summary>
+    internal static bool HasAnyAccountMetadata(JsonNode? result)
+    {
+        if (result is not JsonArray accounts || accounts.Count == 0) return true;
+
+        var sawAccount = false;
+        foreach (var account in accounts)
+        {
+            if (account is not JsonObject) continue;
+            sawAccount = true;
+            var meta = JsVal.AsString(JsVal.Prop(account, "posting_json_metadata"));
+            if (!string.IsNullOrEmpty(meta)) return true;
+        }
+
+        // An all-null array (unknown account) has nothing to prefer either way.
+        return !sawAccount;
+    }
+
     public Task<JsonNode?> GetDynamicGlobalProperties() =>
         Call("condenser_api", "get_dynamic_global_properties", new JsonArray(),
             r => r is JsonObject);
@@ -237,15 +329,19 @@ public NodeUnavailableException(string message, bool advanceImmediately,
 /// </summary>
 public static class HiveClients
 {
+    // techcoderx.com and hiveapi.actifit.io are deliberately absent: both serve
+    // accounts with posting_json_metadata stripped (balances correct, metadata
+    // empty). They are fast, so the latency EWMA ranked them first and the
+    // portfolio engine/chain layers came back empty for everyone. GetAccounts
+    // also routes around such a node at runtime, but keeping them out of the pool
+    // means correctness here does not depend on that fallback firing.
     public static readonly HiveRpcClient Default = new(new[]
     {
         "https://api.hive.blog",
-        "https://techcoderx.com",
         "https://api.deathwing.me",
         "https://rpc.mahdiyari.info",
         "https://hive-api.arcange.eu",
         "https://api.openhive.network",
-        "https://hiveapi.actifit.io",
         "https://hive-api.3speak.tv",
         "https://api.syncad.com",
         "https://api.c0ff33a.uk",